Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Windows/302/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseParallelGC

4 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.lucene.store.TestFileSwitchDirectory

Error Message:
Could not remove the following files (in the order of attempts):    
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001:
 java.nio.file.DirectoryNotEmptyException: 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001
 

Stack Trace:
java.io.IOException: Could not remove the following files (in the order of 
attempts):
   
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001:
 java.nio.file.DirectoryNotEmptyException: 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001

        at __randomizedtesting.SeedInfo.seed([8E750C09AB49F8DA]:0)
        at org.apache.lucene.util.IOUtils.rm(IOUtils.java:329)
        at 
org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:216)
        at 
com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: 
http://127.0.0.1:56628/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: http://127.0.0.1:56628/collection1
        at 
__randomizedtesting.SeedInfo.seed([F8F4332C03B5FC24:70A00CF6AD4991DC]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1582)
        at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 53 more


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
7 threads leaked from SUITE scope at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=13837, 
name=zkCallback-2351-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13699, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-EventThread,
 state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    
3) Thread[id=13700, name=zkCallback-2351-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13697, 
name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13819, 
name=zkCallback-2351-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    6) Thread[id=13838, 
name=zkCallback-2351-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    7) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE 
scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=13837, name=zkCallback-2351-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13699, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-EventThread,
 state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   3) Thread[id=13700, name=zkCallback-2351-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13697, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13819, name=zkCallback-2351-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=13838, name=zkCallback-2351-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([F8F4332C03B5FC24]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=13837, name=zkCallback-2351-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    2) Thread[id=13700, 
name=zkCallback-2351-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    3) Thread[id=13819, 
name=zkCallback-2351-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    4) Thread[id=13838, 
name=zkCallback-2351-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)    5) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=13837, name=zkCallback-2351-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=13700, name=zkCallback-2351-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=13819, name=zkCallback-2351-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=13838, name=zkCallback-2351-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([F8F4332C03B5FC24]:0)




Build Log:
[...truncated 1267 lines...]
   [junit4] Suite: org.apache.lucene.store.TestFileSwitchDirectory
   [junit4] IGNOR/A 0.01s J1 | 
TestFileSwitchDirectory.testFsyncDoesntCreateNewFiles
   [junit4]    > Assumption #1: test only works for FSDirectory subclasses
   [junit4] IGNOR/A 0.01s J1 | TestFileSwitchDirectory.testPendingDeletions
   [junit4]    > Assumption #1: we can only install VirusCheckingFS on an 
FSDirectory
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1290, maxMBSortInHeap=7.642517381374831, 
sim=RandomSimilarity(queryNorm=false): {}, locale=ko-KR, timezone=America/Nassau
   [junit4]   2> NOTE: Windows 10 10.0 x86/Oracle Corporation 1.8.0_144 
(32-bit)/cpus=3,threads=1,free=94052000,total=257687552
   [junit4]   2> NOTE: All tests run in this JVM: [TestMultiFields, 
TestSpanTermQuery, TestOneMergeWrappingMergePolicy, 
TestIndexWriterDeleteByQuery, TestMultiTermQueryRewrites, Test2BPagedBytes, 
TestNGramPhraseQuery, TestBlockPostingsFormat, TestCharTermAttributeImpl, 
TestWildcard, TestIndexWriterDelete, TestTransactions, 
LimitedFiniteStringsIteratorTest, TestRollingBuffer, TestSegmentMerger, 
TestExitableDirectoryReader, Test2BNumericDocValues, TestDocIDMerger, 
TestFixedLengthBytesRefArray, TestDisjunctionMaxQuery, TestLucene70NormsFormat, 
TestIndexWriterConfig, TestPagedBytes, TestParallelTermEnum, 
TestBufferedChecksum, TestPersistentSnapshotDeletionPolicy, TestShardSearching, 
TestSimpleFSLockFactory, TestSimilarity, TestMaxTermFrequency, 
TestInPlaceMergeSorter, TestBytesRefAttImpl, 
TestApproximationSearchEquivalence, TestStringMSBRadixSorter, 
TestLucene50StoredFieldsFormatHighCompression, TestDuelingCodecsAtNight, 
TestPointValues, TestConcurrentMergeScheduler, TestNRTReaderWithThreads, 
TestDirectory, TestMergeSchedulerExternal, TestSortRescorer, 
TestCustomTermFreq, TestLucene50CompoundFormat, TestPrefixInBooleanQuery, 
TestTermRangeQuery, TestBoostQuery, TestDocIdsWriter, 
TestLucene60FieldInfoFormat, TestMultiDocValues, TestNoMergeScheduler, 
TestDeletionPolicy, TestBoolean2, TestLongPostings, TestTermsEnum, 
TestTimSorterWorstCase, TestAutomatonQueryUnicode, TestPackedInts, 
TestPostingsOffsets, TestPrefixCodedTerms, TestBM25Similarity, 
TestPositiveScoresOnlyCollector, TestRollback, TestBooleanScorer, 
TestDoubleValuesSource, TestStressAdvance, TestNumericUtils, 
TestPerSegmentDeletes, TestPerFieldPostingsFormat2, TestPayloads, 
TestSparseFixedBitSet, TestVersion, TestTermsEnum2, TestNot, 
TestForTooMuchCloning, TestBlockPostingsFormat3, TestIndexedDISI, TestCharsRef, 
TestDemo, TestOperations, TestMultiCollector, TestSearchAfter, 
TestDocumentWriter, TestIntroSelector, TestTermdocPerf, TestSmallFloat, 
TestStressNRT, TestUsageTrackingFilterCachingPolicy, TestSloppyMath, 
TestIndexWriterReader, TestFieldMaskingSpanQuery, TestIndexWriterForceMerge, 
TestDocValues, TestFuzzyQuery, TestNRTReaderCleanup, TestTermVectorsWriter, 
TestMergePolicyWrapper, Test2BBKDPoints, TestNormsFieldExistsQuery, 
FuzzyTermOnShortTermsTest, TestFilterDirectoryReader, 
TestIndexWriterFromReader, TestAllFilesCheckIndexHeader, TestByteSlices, 
TestMultiset, TestDeterminism, TestSizeBoundedForceMerge, TestCachingCollector, 
TestDocValuesRewriteMethod, TestSynonymQuery, TestCheckIndex, 
TestMixedDocValuesUpdates, TestIntRangeFieldQueries, 
TestTotalHitCountCollector, TestSubScorerFreqs, TestIndexWriter, 
TestPerFieldDocValuesFormat, TestBytesStore, TestGraphTokenizers, 
TestIndexWriterExceptions, TestBasics, TestFlushByRamOrCountsPolicy, 
TestNRTThreads, TestAtomicUpdate, TestIndexWriterCommit, 
TestDirectoryReaderReopen, TestTimeLimitingCollector, TestSearchWithThreads, 
TestDocumentsWriterStallControl, TestRollingUpdates, 
TestSnapshotDeletionPolicy, TestLevenshteinAutomata, 
TestConsistentFieldNumbers, TestSimpleExplanations, TestPhraseQuery, 
TestOmitNorms, TestComplexExplanations, TestRegexpRandom2, 
TestNRTCachingDirectory, TestParallelCompositeReader, 
TestIndexWriterNRTIsCurrent, TestSimpleSearchEquivalence, 
TestDocValuesIndexing, TestSimilarityBase, TestBooleanMinShouldMatch, 
TestSpans, TestTransactionRollback, TestFlex, TestSpanExplanations, 
TestFastCompressionMode, TestStressIndexing2, TestBytesRefHash, 
TestCrashCausesCorruptIndex, TestScorerPerf, TestExternalCodecs, 
TestNoDeletionPolicy, TestRegexpQuery, TestTopDocsCollector, 
TestUniqueTermCount, TestComplexExplanationsOfNonMatches, TestSortRandom, 
TestParallelReaderEmptyIndex, TestTermScorer, TestTopFieldCollector, 
TestFileSwitchDirectory]
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestFileSwitchDirectory -Dtests.seed=8E750C09AB49F8DA 
-Dtests.slow=true -Dtests.locale=ko-KR -Dtests.timezone=America/Nassau 
-Dtests.asserts=true -Dtests.file.encoding=Cp1252
   [junit4] ERROR   0.00s J1 | TestFileSwitchDirectory (suite) <<<
   [junit4]    > Throwable #1: java.io.IOException: Could not remove the 
following files (in the order of attempts):
   [junit4]    >    
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001:
 java.nio.file.DirectoryNotEmptyException: 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\lucene\build\core\test\J1\temp\lucene.store.TestFileSwitchDirectory_8E750C09AB49F8DA-001\testCopyDestination-001
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([8E750C09AB49F8DA]:0)
   [junit4]    >        at org.apache.lucene.util.IOUtils.rm(IOUtils.java:329)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [322/456 (1!)] on J1 in 2.11s, 43 tests, 1 error, 2 
skipped <<< FAILURES!

[...truncated 11351 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\init-core-data-001
   [junit4]   2> 1123394 WARN  
(SUITE-ChaosMonkeyNothingIsSafeTest-seed#[F8F4332C03B5FC24]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 1123394 INFO  
(SUITE-ChaosMonkeyNothingIsSafeTest-seed#[F8F4332C03B5FC24]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1123400 INFO  
(SUITE-ChaosMonkeyNothingIsSafeTest-seed#[F8F4332C03B5FC24]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1123400 INFO  
(SUITE-ChaosMonkeyNothingIsSafeTest-seed#[F8F4332C03B5FC24]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1123400 INFO  
(SUITE-ChaosMonkeyNothingIsSafeTest-seed#[F8F4332C03B5FC24]-worker) [    ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1123404 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1123404 INFO  (Thread-3288) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1123404 INFO  (Thread-3288) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 1123409 ERROR (Thread-3288) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1123505 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ZkTestServer start zk server on port:56576
   [junit4]   2> 1123515 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15fab41c1200000, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1123527 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 1123529 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\schema15.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 1123535 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1123537 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 1123540 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 1123542 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 1123544 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1123546 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1123549 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1123551 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1123553 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractZkTestCase put 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\core\src\test-files\solr\collection1\conf\synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 1123555 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly 
asked otherwise
   [junit4]   2> 1124044 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1124045 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@bce3d9{/,null,AVAILABLE}
   [junit4]   2> 1124046 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@33be0c{HTTP/1.1,[http/1.1]}{127.0.0.1:56583}
   [junit4]   2> 1124046 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server Started @1131667ms
   [junit4]   2> 1124047 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\tempDir-001/control/data,
 replicaType=NRT, hostContext=/, hostPort=56583, 
coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\control-001\cores}
   [junit4]   2> 1124047 ERROR 
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1124047 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.2.0
   [junit4]   2> 1124047 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1124047 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1124047 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-11-11T13:25:34.752Z
   [junit4]   2> 1124052 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15fab41c1200002, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1124052 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1124052 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\control-001\solr.xml
   [junit4]   2> 1124061 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay 
is ignored
   [junit4]   2> 1124061 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter 
autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1124063 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1124068 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56576/solr
   [junit4]   2> 1124205 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1124206 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.c.OverseerElectionContext I am going to be the 
leader 127.0.0.1:56583_
   [junit4]   2> 1124208 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.c.Overseer Overseer 
(id=98986015726764036-127.0.0.1:56583_-n_0000000000) starting
   [junit4]   2> 1124217 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:56583_
   [junit4]   2> 1124220 INFO  
(OverseerStateUpdate-98986015726764036-127.0.0.1:56583_-n_0000000000) 
[n:127.0.0.1:56583_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1124451 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1124462 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1124462 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1124465 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56583_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions 
underneath 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\control-001\cores
   [junit4]   2> 1124503 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1124504 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56576/solr ready
   [junit4]   2> 1124506 INFO  (qtp29940148-13657) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56583_&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1124510 INFO  
(OverseerThreadFactory-5831-thread-1-processing-n:127.0.0.1:56583_) 
[n:127.0.0.1:56583_    ] o.a.s.c.CreateCollectionCmd Create collection 
control_collection
   [junit4]   2> 1124619 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1124620 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 1124726 INFO  
(zkCallback-2343-thread-1-processing-n:127.0.0.1:56583_) [n:127.0.0.1:56583_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1125648 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 1125662 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1125740 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1125762 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' 
using configuration from collection control_collection, trusted=true
   [junit4]   2> 1125763 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.control_collection.shard1.replica_n1' (registry 
'solr.core.control_collection.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1125763 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1125764 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore 
at 
[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\control-001\cores\control_collection_shard1_replica_n1],
 
dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\control-001\cores\control_collection_shard1_replica_n1\data\]
   [junit4]   2> 1125771 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=43, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=69.384765625, 
floorSegmentMB=1.533203125, forceMergeDeletesPctAllowed=29.066107936980956, 
segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1125776 WARN  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 1125860 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 1125861 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1125862 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 1125862 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1125863 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=24, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=1.19921875, 
floorSegmentMB=1.37109375, forceMergeDeletesPctAllowed=10.394928655465021, 
segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.8805165702684776
   [junit4]   2> 1125865 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@10f55cb[control_collection_shard1_replica_n1] main]
   [junit4]   2> 1125866 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1125866 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1125867 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1125868 INFO  
(searcherExecutor-5834-thread-1-processing-n:127.0.0.1:56583_ 
x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) 
[n:127.0.0.1:56583_ c:control_collection s:shard1  
x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore 
[control_collection_shard1_replica_n1] Registered new searcher 
Searcher@10f55cb[control_collection_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1125869 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1583776254209818624
   [junit4]   2> 1125874 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1125875 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1125875 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:56583/control_collection_shard1_replica_n1/
   [junit4]   2> 1125875 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1125875 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.SyncStrategy 
http://127.0.0.1:56583/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1125875 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 1125878 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:56583/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 1125981 INFO  
(zkCallback-2343-thread-2-processing-n:127.0.0.1:56583_) [n:127.0.0.1:56583_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126031 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1126034 INFO  (qtp29940148-13659) [n:127.0.0.1:56583_ 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1416
   [junit4]   2> 1126039 INFO  (qtp29940148-13657) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 1126135 INFO  
(zkCallback-2343-thread-2-processing-n:127.0.0.1:56583_) [n:127.0.0.1:56583_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/control_collection/state.json] for collection 
[control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1126512 INFO  
(OverseerCollectionConfigSetProcessor-98986015726764036-127.0.0.1:56583_-n_0000000000)
 [n:127.0.0.1:56583_    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 1127039 INFO  (qtp29940148-13657) [n:127.0.0.1:56583_    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56583_&wt=javabin&version=2}
 status=0 QTime=2532
   [junit4]   2> 1127040 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15fab41c1200005, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1127050 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1127051 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56576/solr ready
   [junit4]   2> 1127051 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection 
loss:false
   [junit4]   2> 1127053 INFO  (qtp29940148-13662) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1127056 INFO  
(OverseerThreadFactory-5831-thread-2-processing-n:127.0.0.1:56583_) 
[n:127.0.0.1:56583_    ] o.a.s.c.CreateCollectionCmd Create collection 
collection1
   [junit4]   2> 1127057 WARN  
(OverseerThreadFactory-5831-thread-2-processing-n:127.0.0.1:56583_) 
[n:127.0.0.1:56583_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a 
collection (collection1) without cores.
   [junit4]   2> 1127261 INFO  (qtp29940148-13662) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 1127261 INFO  (qtp29940148-13662) [n:127.0.0.1:56583_    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2}
 status=0 QTime=208
   [junit4]   2> 1127744 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001
 of type TLOG
   [junit4]   2> 1127745 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1127746 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@140d02b{/,null,AVAILABLE}
   [junit4]   2> 1127747 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@c5dd0{HTTP/1.1,[http/1.1]}{127.0.0.1:56610}
   [junit4]   2> 1127747 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server Started @1135368ms
   [junit4]   2> 1127747 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\tempDir-001/jetty1,
 replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=56610, 
coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001\cores}
   [junit4]   2> 1127748 ERROR 
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1127748 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.2.0
   [junit4]   2> 1127748 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1127748 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1127748 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-11-11T13:25:38.453Z
   [junit4]   2> 1127754 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1127754 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001\solr.xml
   [junit4]   2> 1127762 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay 
is ignored
   [junit4]   2> 1127762 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter 
autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1127762 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1127767 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56576/solr
   [junit4]   2> 1127771 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15fab41c1200008, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1127782 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1127787 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1127791 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:56610_
   [junit4]   2> 1127793 INFO  (zkCallback-2351-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1127793 INFO  
(zkCallback-2356-thread-1-processing-n:127.0.0.1:56610_) [n:127.0.0.1:56610_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1127793 INFO  
(zkCallback-2343-thread-2-processing-n:127.0.0.1:56583_) [n:127.0.0.1:56583_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1128020 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1128036 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1128037 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1128040 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56610_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions 
underneath 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001\cores
   [junit4]   2> 1128089 INFO  (qtp29940148-13655) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params 
node=127.0.0.1:56610_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1128094 INFO  
(OverseerCollectionConfigSetProcessor-98986015726764036-127.0.0.1:56583_-n_0000000000)
 [n:127.0.0.1:56583_    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 1128095 INFO  
(OverseerThreadFactory-5831-thread-3-processing-n:127.0.0.1:56583_) 
[n:127.0.0.1:56583_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56610_ 
for creating new replica
   [junit4]   2> 1128101 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 1128102 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 1129132 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 1129151 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema 
[collection1_shard1_replica_t21] Schema name=test
   [junit4]   2> 1129233 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema 
Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1129256 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1_shard1_replica_t21' using configuration from 
collection collection1, trusted=true
   [junit4]   2> 1129257 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.collection1.shard1.replica_t21' (registry 
'solr.core.collection1.shard1.replica_t21') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1129257 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 1129257 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore 
[[collection1_shard1_replica_t21] ] Opening new SolrCore at 
[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001\cores\collection1_shard1_replica_t21],
 
dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-1-001\cores\collection1_shard1_replica_t21\data\]
   [junit4]   2> 1129264 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=43, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=69.384765625, 
floorSegmentMB=1.533203125, forceMergeDeletesPctAllowed=29.066107936980956, 
segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1129271 WARN  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 1129356 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1129356 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1129358 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker 
Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 1129358 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1129360 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=24, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=1.19921875, 
floorSegmentMB=1.37109375, forceMergeDeletesPctAllowed=10.394928655465021, 
segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.8805165702684776
   [junit4]   2> 1129360 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1e13d1f[collection1_shard1_replica_t21] main]
   [junit4]   2> 1129362 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1129362 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1129364 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1129366 INFO  
(searcherExecutor-5845-thread-1-processing-n:127.0.0.1:56610_ 
x:collection1_shard1_replica_t21 s:shard1 c:collection1) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SolrCore 
[collection1_shard1_replica_t21] Registered new searcher 
Searcher@1e13d1f[collection1_shard1_replica_t21] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1129366 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1583776257876688896
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:56610/collection1_shard1_replica_t21/
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy 
http://127.0.0.1:56610/collection1_shard1_replica_t21/ has no replicas
   [junit4]   2> 1129371 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 1129372 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController 
collection1_shard1_replica_t21 stopping background replication from leader
   [junit4]   2> 1129376 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:56610/collection1_shard1_replica_t21/ shard1
   [junit4]   2> 1129528 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.c.ZkController 
I am the leader, no recovery necessary
   [junit4]   2> 1129531 INFO  (qtp2312421-13704) [n:127.0.0.1:56610_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t21] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=1429
   [junit4]   2> 1129534 INFO  (qtp29940148-13655) [n:127.0.0.1:56583_    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={node=127.0.0.1:56610_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2}
 status=0 QTime=1444
   [junit4]   2> 1130015 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001
 of type TLOG
   [junit4]   2> 1130016 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1130016 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1b4e1d1{/,null,AVAILABLE}
   [junit4]   2> 1130018 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@ca3af2{HTTP/1.1,[http/1.1]}{127.0.0.1:56628}
   [junit4]   2> 1130018 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.e.j.s.Server Started @1137639ms
   [junit4]   2> 1130018 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\tempDir-001/jetty2,
 replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=56628, 
coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001\cores}
   [junit4]   2> 1130018 ERROR 
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1130019 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.2.0
   [junit4]   2> 1130019 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1130019 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1130019 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-11-11T13:25:40.724Z
   [junit4]   2> 1130023 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1130023 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001\solr.xml
   [junit4]   2> 1130030 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay 
is ignored
   [junit4]   2> 1130030 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig Configuration parameter 
autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1130031 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1130036 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56576/solr
   [junit4]   2> 1130059 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 1130062 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1130066 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:56628_
   [junit4]   2> 1130067 INFO  (zkCallback-2351-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1130067 INFO  
(zkCallback-2356-thread-1-processing-n:127.0.0.1:56610_) [n:127.0.0.1:56610_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1130068 INFO  
(zkCallback-2343-thread-1-processing-n:127.0.0.1:56583_) [n:127.0.0.1:56583_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1130068 INFO  
(zkCallback-2362-thread-1-processing-n:127.0.0.1:56628_) [n:127.0.0.1:56628_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1130098 INFO  
(OverseerCollectionConfigSetProcessor-98986015726764036-127.0.0.1:56583_-n_0000000000)
 [n:127.0.0.1:56583_    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 1130266 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1130279 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1130279 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1130281 INFO  
(TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]) 
[n:127.0.0.1:56628_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions 
underneath 
C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001\cores
   [junit4]   2> 1130328 INFO  (qtp29940148-13658) [n:127.0.0.1:56583_    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params 
node=127.0.0.1:56628_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1130333 INFO  
(OverseerThreadFactory-5831-thread-4-processing-n:127.0.0.1:56583_) 
[n:127.0.0.1:56583_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56628_ 
for creating new replica
   [junit4]   2> 1130336 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 1130338 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 
transient cores
   [junit4]   2> 1131362 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 1131375 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema 
[collection1_shard1_replica_t23] Schema name=test
   [junit4]   2> 1131451 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema 
Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1131474 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1_shard1_replica_t23' using configuration from 
collection collection1, trusted=true
   [junit4]   2> 1131474 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.collection1.shard1.replica_t23' (registry 
'solr.core.collection1.shard1.replica_t23') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@95eaf0
   [junit4]   2> 1131475 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 1131475 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore 
[[collection1_shard1_replica_t23] ] Opening new SolrCore at 
[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001\cores\collection1_shard1_replica_t23],
 
dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-7.x-Windows\solr\build\solr-core\test\J0\temp\solr.cloud.ChaosMonkeyNothingIsSafeTest_F8F4332C03B5FC24-001\shard-2-001\cores\collection1_shard1_replica_t23\data\]
   [junit4]   2> 1131481 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=43, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=69.384765625, 
floorSegmentMB=1.533203125, forceMergeDeletesPctAllowed=29.066107936980956, 
segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1131486 WARN  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 1131571 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1131571 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1131573 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker 
Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 1131573 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1131574 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=24, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=1.19921875, 
floorSegmentMB=1.37109375, forceMergeDeletesPctAllowed=10.394928655465021, 
segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.8805165702684776
   [junit4]   2> 1131575 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1ed4bee[collection1_shard1_replica_t23] main]
   [junit4]   2> 1131577 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1131578 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1131578 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1131579 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1583776260197187584
   [junit4]   2> 1131580 INFO  
(searcherExecutor-5856-thread-1-processing-n:127.0.0.1:56628_ 
x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.SolrCore 
[collection1_shard1_replica_t23] Registered new searcher 
Searcher@1ed4bee[collection1_shard1_replica_t23] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1131583 INFO  (qtp17996096-13733) [n:127.0.0.1:56628_ 
c:collection1 s:shard1  x:collection1_shard1_replica_t23] o.a.s.c.ZkController 
Core needs to recover:collection1_shard1_replica_t23
   [junit4]   2> 1131583 INFO  
(updateExecutor-2359-thread-1-processing-n:127.0.0.1:56628_ 
x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:56628_ 
c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] 
o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1131583 INFO  
(recoveryExecutor-2360-thread-1-processing-n:127.0.0.1:56628_ 
x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) 
[n:127.0.0.1:56628_ c:collection1 s

[...truncated too long message...]

1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=13699, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-EventThread,
 state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   [junit4]    >    3) Thread[id=13700, name=zkCallback-2351-thread-1, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=13697, name=Connection evictor, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=13819, name=zkCallback-2351-thread-2, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=13838, name=zkCallback-2351-thread-4, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([F8F4332C03B5FC24]:0)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=13837, name=zkCallback-2351-thread-3, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=13700, name=zkCallback-2351-thread-1, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=13819, name=zkCallback-2351-thread-2, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=13838, name=zkCallback-2351-thread-4, 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=13698, 
name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[F8F4332C03B5FC24]-SendThread(127.0.0.1:56576),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([F8F4332C03B5FC24]:0)
   [junit4] Completed [305/748 (1!)] on J0 in 79.63s, 1 test, 3 errors <<< 
FAILURES!

[...truncated 44492 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to