ZooKeeper_branch35_jdk8 - Build # 541 - Still Failing

[Apache Jenkins Server]

See https://builds.apache.org/job/ZooKeeper_branch35_jdk8/541/

###################################################################################
########################## LAST 60 LINES OF THE CONSOLE 
###########################
[...truncated 54.10 MB...]
    [junit]     at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:373)
    [junit]     at 
org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:81)
    [junit]     at 
org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36)
    [junit]     at 
org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:779)
    [junit]     at 
org.jboss.netty.channel.SimpleChannelHandler.closeRequested(SimpleChannelHandler.java:334)
    [junit]     at 
org.jboss.netty.channel.SimpleChannelHandler.handleDownstream(SimpleChannelHandler.java:260)
    [junit]     at 
org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:591)
    [junit]     at 
org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:582)
    [junit]     at org.jboss.netty.channel.Channels.close(Channels.java:812)
    [junit]     at 
org.jboss.netty.channel.AbstractChannel.close(AbstractChannel.java:206)
    [junit]     at 
org.apache.zookeeper.server.NettyServerCnxn.close(NettyServerCnxn.java:118)
    [junit]     at 
org.apache.zookeeper.server.NettyServerCnxn.sendBuffer(NettyServerCnxn.java:221)
    [junit]     at 
org.apache.zookeeper.server.NettyServerCnxn.sendCloseSession(NettyServerCnxn.java:460)
    [junit]     at 
org.apache.zookeeper.server.FinalRequestProcessor.processRequest(FinalRequestProcessor.java:461)
    [junit]     at 
org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:182)
    [junit]     at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:113)
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  [main:ZooKeeper@1331] - 
Session: 0x1047efd6adf0000 closed
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  
[main:JUnit4ZKTestRunner$LoggedInvokeMethod@82] - Memory used 219437
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  
[main:JUnit4ZKTestRunner$LoggedInvokeMethod@87] - Number of threads 2428
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  
[main:JUnit4ZKTestRunner$LoggedInvokeMethod@102] - FINISHED TEST METHOD 
testWatcherAutoResetWithLocal
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  [main:ClientBase@567] - 
tearDown starting
    [junit] 2017-05-29 12:30:41,467 [myid:] - INFO  [main:ClientBase@537] - 
STOPPING server
    [junit] 2017-05-29 12:30:41,467 [myid:] - INFO  
[main:NettyServerCnxnFactory@464] - shutdown called 0.0.0.0/0.0.0.0:30319
    [junit] 2017-05-29 12:30:41,466 [myid:] - INFO  
[main-EventThread:ClientCnxn$EventThread@513] - EventThread shut down for 
session: 0x1047efd6adf0000
    [junit] 2017-05-29 12:30:41,472 [myid:] - INFO  [main:ZooKeeperServer@541] 
- shutting down
    [junit] 2017-05-29 12:30:41,472 [myid:] - ERROR [main:ZooKeeperServer@505] 
- ZKShutdownHandler is not registered, so ZooKeeper server won't take any 
action on ERROR or SHUTDOWN server state changes
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  
[main:SessionTrackerImpl@232] - Shutting down
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  
[main:PrepRequestProcessor@1004] - Shutting down
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  
[main:SyncRequestProcessor@191] - Shutting down
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  [ProcessThread(sid:0 
cport:30319)::PrepRequestProcessor@156] - PrepRequestProcessor exited loop!
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  
[SyncThread:0:SyncRequestProcessor@169] - SyncRequestProcessor exited!
    [junit] 2017-05-29 12:30:41,473 [myid:] - INFO  
[main:FinalRequestProcessor@481] - shutdown of request processor complete
    [junit] 2017-05-29 12:30:41,483 [myid:] - INFO  [main:MBeanRegistry@128] - 
Unregister MBean 
[org.apache.ZooKeeperService:name0=StandaloneServer_port30319,name1=InMemoryDataTree]
    [junit] 2017-05-29 12:30:41,483 [myid:] - INFO  [main:MBeanRegistry@128] - 
Unregister MBean [org.apache.ZooKeeperService:name0=StandaloneServer_port30319]
    [junit] 2017-05-29 12:30:41,528 [myid:] - INFO  
[main:FourLetterWordMain@85] - connecting to 127.0.0.1 30319
    [junit] 2017-05-29 12:30:41,529 [myid:] - INFO  [main:JMXEnv@146] - 
ensureOnly:[]
    [junit] 2017-05-29 12:30:41,540 [myid:] - INFO  [main:ClientBase@592] - 
fdcount after test is: 7135 at start it was 7139
    [junit] 2017-05-29 12:30:41,540 [myid:] - INFO  [main:ZKTestCase$1@68] - 
SUCCEEDED testWatcherAutoResetWithLocal
    [junit] 2017-05-29 12:30:41,540 [myid:] - INFO  [main:ZKTestCase$1@63] - 
FINISHED testWatcherAutoResetWithLocal
    [junit] Tests run: 103, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 
475.394 sec, Thread: 8, Class: org.apache.zookeeper.test.NioNettySuiteTest
    [junit] 2017-05-29 12:30:41,772 [myid:127.0.0.1:30196] - INFO  
[main-SendThread(127.0.0.1:30196):ClientCnxn$SendThread@1113] - Opening socket 
connection to server 127.0.0.1/127.0.0.1:30196. Will not attempt to 
authenticate using SASL (unknown error)
    [junit] 2017-05-29 12:30:41,773 [myid:127.0.0.1:30196] - WARN  
[main-SendThread(127.0.0.1:30196):ClientCnxn$SendThread@1235] - Session 
0x1047efa01b90000 for server 127.0.0.1/127.0.0.1:30196, unexpected error, 
closing socket connection and attempting reconnect
    [junit] java.net.ConnectException: Connection refused
    [junit]     at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
    [junit]     at 
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
    [junit]     at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:357)
    [junit]     at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1214)

fail.build.on.test.failure:

BUILD FAILED
/home/jenkins/jenkins-slave/workspace/ZooKeeper_branch35_jdk8/build.xml:1219: 
The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/ZooKeeper_branch35_jdk8/build.xml:1223: 
Tests failed!

Total time: 10 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



###################################################################################
############################## FAILED TESTS (if any) 
##############################
50 tests failed.
FAILED:  
org.apache.zookeeper.server.quorum.ReconfigRecoveryTest.testCurrentServersAreObserversInNextConfig

Error Message:
waiting for server 0 being up

Stack Trace:
junit.framework.AssertionFailedError: waiting for server 0 being up
        at 
org.apache.zookeeper.server.quorum.ReconfigRecoveryTest.testCurrentServersAreObserversInNextConfig(ReconfigRecoveryTest.java:217)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)


FAILED:  
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testResyncByDiffAfterFollowerCrashes

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger.start(FastLeaderElection.java:520)
        at 
org.apache.zookeeper.server.quorum.FastLeaderElection.start(FastLeaderElection.java:637)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.createElectionAlgorithm(QuorumPeer.java:970)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.startLeaderElection(QuorumPeer.java:856)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.start(QuorumPeer.java:780)
        at org.apache.zookeeper.test.QuorumUtil.start(QuorumUtil.java:211)
        at org.apache.zookeeper.test.QuorumUtil.restart(QuorumUtil.java:215)
        at 
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testResyncByDiffAfterFollowerCrashes(FollowerResyncConcurrencyTest.java:492)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)


FAILED:  
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testFollowerSendsLastZxid

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.start(NIOServerCnxnFactory.java:751)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.startServerCnxnFactory(QuorumPeer.java:1620)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.start(QuorumPeer.java:773)
        at org.apache.zookeeper.test.QuorumUtil.start(QuorumUtil.java:211)
        at org.apache.zookeeper.test.QuorumUtil.startAll(QuorumUtil.java:145)
        at 
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testFollowerSendsLastZxid(FollowerResyncConcurrencyTest.java:672)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)


FAILED:  
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testResyncBySnapThenDiffAfterFollowerCrashes

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ClientCommunicatorAdmin.<init>(ClientCommunicatorAdmin.java:45)
        at 
javax.management.remote.rmi.RMIConnector$RMIClientCommunicatorAdmin.<init>(RMIConnector.java:1493)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:339)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.followerResyncCrashTest(FollowerResyncConcurrencyTest.java:199)
        at 
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testResyncBySnapThenDiffAfterFollowerCrashes(FollowerResyncConcurrencyTest.java:176)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)


FAILED:  
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testFollowerWatcherResync

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.FollowerResyncConcurrencyTest.testFollowerWatcherResync(FollowerResyncConcurrencyTest.java:713)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ChrootClientTest.testMutipleWatcherObjs

Error Message:
Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNetty

Stack Trace:
java.io.IOException: Couldn't instantiate 
org.apache.zookeeper.ClientCnxnSocketNetty
        at 
org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:2976)
        at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:880)
        at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:798)
        at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:938)
        at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:685)
        at 
org.apache.zookeeper.admin.ZooKeeperAdmin.<init>(ZooKeeperAdmin.java:77)
        at 
org.apache.zookeeper.TestableZooKeeper.<init>(TestableZooKeeper.java:36)
        at 
org.apache.zookeeper.test.ClientBase.createClient(ClientBase.java:209)
        at 
org.apache.zookeeper.test.ClientBase.createClient(ClientBase.java:201)
        at 
org.apache.zookeeper.test.ClientBase.createClient(ClientBase.java:186)
        at 
org.apache.zookeeper.test.ChrootClientTest.setUp(ChrootClientTest.java:41)
Caused by: java.lang.reflect.InvocationTargetException
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at 
org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:2973)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory.<init>(NioClientSocketChannelFactory.java:151)
        at 
org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory.<init>(NioClientSocketChannelFactory.java:116)
        at 
org.apache.zookeeper.ClientCnxnSocketNetty.<init>(ClientCnxnSocketNetty.java:72)


FAILED:  org.apache.zookeeper.test.ClientTest.testTryWithResources

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.apache.zookeeper.server.ZooKeeperServer.startSessionTracker(ZooKeeperServer.java:479)
        at 
org.apache.zookeeper.server.ZooKeeperServer.startup(ZooKeeperServer.java:451)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.startup(NettyServerCnxnFactory.java:511)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.startup(ServerCnxnFactory.java:98)
        at 
org.apache.zookeeper.test.ClientBase.startServerInstance(ClientBase.java:387)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:493)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testACLs

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.apache.zookeeper.server.ZooKeeperServer.startSessionTracker(ZooKeeperServer.java:479)
        at 
org.apache.zookeeper.server.ZooKeeperServer.startup(ZooKeeperServer.java:451)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.startup(NettyServerCnxnFactory.java:511)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.startup(ServerCnxnFactory.java:98)
        at 
org.apache.zookeeper.test.ClientBase.startServerInstance(ClientBase.java:387)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:493)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testPing

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at 
org.apache.zookeeper.test.ClientBase.createNewServerInstance(ClientBase.java:412)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:491)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testClientCleanup

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at 
org.apache.zookeeper.test.ClientBase.createNewServerInstance(ClientBase.java:412)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:491)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testClientwithoutWatcherObj

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
sun.rmi.transport.ObjectTable.incrementKeepAliveCount(ObjectTable.java:281)
        at sun.rmi.transport.ObjectTable.putTarget(ObjectTable.java:196)
        at sun.rmi.transport.Transport.exportObject(Transport.java:106)
        at 
sun.rmi.transport.tcp.TCPTransport.exportObject(TCPTransport.java:260)
        at sun.rmi.transport.tcp.TCPEndpoint.exportObject(TCPEndpoint.java:411)
        at sun.rmi.transport.LiveRef.exportObject(LiveRef.java:147)
        at 
sun.rmi.server.UnicastServerRef.exportObject(UnicastServerRef.java:234)
        at 
java.rmi.server.UnicastRemoteObject.exportObject(UnicastRemoteObject.java:383)
        at 
java.rmi.server.UnicastRemoteObject.exportObject(UnicastRemoteObject.java:346)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.export(RMIJRMPServerImpl.java:148)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.export(RMIJRMPServerImpl.java:124)
        at 
javax.management.remote.rmi.RMIConnectorServer.start(RMIConnectorServer.java:404)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:54)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)


FAILED:  org.apache.zookeeper.test.ClientTest.testSequentialNodeData

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ClientTest.testSequentialNodeNames

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ClientTest.testClientWithWatcherObj

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at 
org.apache.zookeeper.test.ClientBase.createNewServerInstance(ClientBase.java:412)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:491)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testTestability

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at 
org.apache.zookeeper.test.ClientBase.createNewServerInstance(ClientBase.java:412)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:491)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testNullAuthId

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at 
org.apache.zookeeper.test.ClientBase.createNewServerInstance(ClientBase.java:412)
        at org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:491)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:484)


FAILED:  org.apache.zookeeper.test.ClientTest.testNonExistingOpCode

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ClientCommunicatorAdmin.<init>(ClientCommunicatorAdmin.java:45)
        at 
javax.management.remote.rmi.RMIConnector$RMIClientCommunicatorAdmin.<init>(RMIConnector.java:1493)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:339)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)


FAILED:  org.apache.zookeeper.test.ClientTest.testDeleteWithChildren

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ClientTest.testPathValidation

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ClientTest.testLargeNodeData

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ClientTest.testMutipleWatcherObjs

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ClientCommunicatorAdmin.<init>(ClientCommunicatorAdmin.java:45)
        at 
javax.management.remote.rmi.RMIConnector$RMIClientCommunicatorAdmin.<init>(RMIConnector.java:1493)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:339)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)


FAILED:  org.apache.zookeeper.test.FourLetterWordsTest.testFourLetterWords

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)


FAILED:  org.apache.zookeeper.test.FourLetterWordsTest.testSetTraceMask

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)


FAILED:  org.apache.zookeeper.test.FourLetterWordsTest.testValidateConsOutput

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)


FAILED:  org.apache.zookeeper.test.FourLetterWordsTest.testValidateSocketTimeout

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)


FAILED:  org.apache.zookeeper.test.FourLetterWordsTest.testValidateStatOutput

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)


FAILED:  org.apache.zookeeper.test.NullDataTest.testNullData

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
        at org.apache.zookeeper.test.NullDataTest.setUp(NullDataTest.java:43)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testQuorumSystemChange

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testQuorumSystemChange(ReconfigTest.java:816)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testPortChange

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testPortChange(ReconfigTest.java:589)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPortLeader

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPort(ReconfigTest.java:722)
        at 
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPortLeader(ReconfigTest.java:718)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.zookeeper.test.ReconfigTest.testInitialConfigHasPositiveVersion

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testInitialConfigHasPositiveVersion(ReconfigTest.java:893)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testRemoveAddOne

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testRemoveAddOne(ReconfigTest.java:250)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testRemoveAddTwo

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testRemoveAddTwo(ReconfigTest.java:332)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPortFollower

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPort(ReconfigTest.java:722)
        at 
org.apache.zookeeper.test.ReconfigTest.testPortChangeToBlockedPortFollower(ReconfigTest.java:714)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testJMXBeanAfterRemoveAddOne

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testJMXBeanAfterRemoveAddOne(ReconfigTest.java:912)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testRoleChange

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testRoleChange(ReconfigTest.java:512)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testBulkReconfig

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testBulkReconfig(ReconfigTest.java:432)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testJMXBeanAfterRoleChange

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testJMXBeanAfterRoleChange(ReconfigTest.java:991)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.ReconfigTest.testRemoveOneAsynchronous

Error Message:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.lang.RuntimeException: java.rmi.ServerError: Error occurred in server 
thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:122)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:127)
        at 
org.apache.zookeeper.test.ReconfigTest.testRemoveOneAsynchronous(ReconfigTest.java:471)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)
Caused by: java.rmi.ServerError: Error occurred in server thread; nested 
exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.QuorumUtil.<init>(QuorumUtil.java:90)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.SessionTest.testSessionTimeout

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  org.apache.zookeeper.test.SessionTest.testMinMaxSessionTimeout

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  org.apache.zookeeper.test.SessionTest.testSessionMove

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  org.apache.zookeeper.test.SessionTest.testSessionTimeoutAccess

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  org.apache.zookeeper.test.SessionTest.testSession

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  
org.apache.zookeeper.test.SessionTest.testSessionStateNoDupStateReporting

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52)
        at 
org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28)
        at 
org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39)
        at 
org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:33)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:149)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:131)
        at 
org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory.<init>(NioServerSocketChannelFactory.java:115)
        at 
org.apache.zookeeper.server.NettyServerCnxnFactory.<init>(NettyServerCnxnFactory.java:332)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:133)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:153)
        at 
org.apache.zookeeper.server.ServerCnxnFactory.createFactory(ServerCnxnFactory.java:147)
        at org.apache.zookeeper.test.SessionTest.setUp(SessionTest.java:77)


FAILED:  org.apache.zookeeper.test.WatcherTest.testWatchAutoResetWithPending

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
        at org.apache.zookeeper.test.WatcherTest.setUp(WatcherTest.java:76)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.WatcherTest.testWatcherCorrectness

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
        at org.apache.zookeeper.test.WatcherTest.setUp(WatcherTest.java:76)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.zookeeper.test.WatcherTest.testWatcherAutoResetDisabledWithLocal

Error Message:
Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread

Stack Trace:
java.rmi.ServerError: Error occurred in server thread; nested exception is: 
        java.lang.OutOfMemoryError: unable to create new native thread
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:374)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at 
sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
        at 
sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
        at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
        at javax.management.remote.rmi.RMIServerImpl_Stub.newClient(Unknown 
Source)
        at 
javax.management.remote.rmi.RMIConnector.getConnection(RMIConnector.java:2430)
        at 
javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:308)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
        at 
javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:229)
        at org.apache.zookeeper.test.JMXEnv.setUp(JMXEnv.java:58)
        at org.apache.zookeeper.test.ClientBase.setUp(ClientBase.java:478)
        at org.apache.zookeeper.test.WatcherTest.setUp(WatcherTest.java:76)
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
com.sun.jmx.remote.internal.ServerCommunicatorAdmin.<init>(ServerCommunicatorAdmin.java:50)
        at 
javax.management.remote.rmi.RMIConnectionImpl$RMIServerCommunicatorAdmin.<init>(RMIConnectionImpl.java:1321)
        at 
javax.management.remote.rmi.RMIConnectionImpl.<init>(RMIConnectionImpl.java:164)
        at 
javax.management.remote.rmi.RMIJRMPServerImpl.makeClient(RMIJRMPServerImpl.java:207)
        at 
javax.management.remote.rmi.RMIServerImpl.doNewClient(RMIServerImpl.java:250)
        at 
javax.management.remote.rmi.RMIServerImpl.newClient(RMIServerImpl.java:199)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.zookeeper.test.QuorumMajorityTest.testMajQuorums

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.start(NIOServerCnxnFactory.java:746)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.startServerCnxnFactory(QuorumPeer.java:1620)
        at 
org.apache.zookeeper.server.quorum.QuorumPeer.start(QuorumPeer.java:773)
        at 
org.apache.zookeeper.test.QuorumBase.startServers(QuorumBase.java:214)
        at org.apache.zookeeper.test.QuorumBase.setUp(QuorumBase.java:120)
        at 
org.apache.zookeeper.test.QuorumMajorityTest.testMajQuorums(QuorumMajorityTest.java:64)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)


FAILED:  org.apache.zookeeper.ZooKeeperTest.testCreateNodeWithoutData

Error Message:
Path must start with / character

Stack Trace:
org.apache.zookeeper.cli.MalformedPathException: Path must start with / 
character
        at org.apache.zookeeper.cli.CreateCommand.exec(CreateCommand.java:122)
        at 
org.apache.zookeeper.ZooKeeperMain.processZKCmd(ZooKeeperMain.java:655)
        at 
org.apache.zookeeper.ZooKeeperTest.testCreateNodeWithoutData(ZooKeeperTest.java:293)
        at 
org.apache.zookeeper.JUnit4ZKTestRunner$LoggedInvokeMethod.evaluate(JUnit4ZKTestRunner.java:79)