Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1461/
6 tests failed. FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test Error Message: Timeout occured while waiting response from server at: http://127.0.0.1:37040/ve_ive/ko/collection1 Stack Trace: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37040/ve_ive/ko/collection1 at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1:E898C48C3E599219]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1591) at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:212) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ... 53 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest Error Message: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=25718, name=zkCallback-3077-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=24716, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=24742, name=zkCallback-3077-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=24717, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-SendThread(127.0.0.1:34454), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=25672, name=zkCallback-3077-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=26068, name=zkCallback-3077-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=24718, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 8) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=25718, name=zkCallback-3077-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=24716, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=24742, name=zkCallback-3077-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=24717, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-SendThread(127.0.0.1:34454), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=25672, name=zkCallback-3077-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=26068, name=zkCallback-3077-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=24718, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 8) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=25718, name=zkCallback-3077-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=24742, name=zkCallback-3077-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=24717, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-SendThread(127.0.0.1:34454), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=25672, name=zkCallback-3077-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=26068, name=zkCallback-3077-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=25718, name=zkCallback-3077-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=24742, name=zkCallback-3077-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=24717, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-SendThread(127.0.0.1:34454), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=25672, name=zkCallback-3077-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=26068, name=zkCallback-3077-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1]:0) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate Error Message: The trigger did not fire at all Stack Trace: java.lang.AssertionError: The trigger did not fire at all at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1:3D84E5DF5F6359AE]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling Error Message: Both triggers should have fired by now Stack Trace: java.lang.AssertionError: Both triggers should have fired by now at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1:9BEE5373420F1C73]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling(TestTriggerIntegration.java:214) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory Error Message: expected:<5> but was:<0> Stack Trace: java.lang.AssertionError: expected:<5> but was:<0> at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1:D305FAB2AED00E6]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.failNotEquals(Assert.java:647) at org.junit.Assert.assertEquals(Assert.java:128) at org.junit.Assert.assertEquals(Assert.java:472) at org.junit.Assert.assertEquals(Assert.java:456) at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12597 lines...] [junit4] Suite: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_60CCFB5690A5FFE1-001/init-core-data-001 [junit4] 2> 538842 WARN (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2 [junit4] 2> 538842 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 538844 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 538844 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 538845 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_60CCFB5690A5FFE1-001/tempDir-001 [junit4] 2> 538845 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 538845 INFO (Thread-241) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 538845 INFO (Thread-241) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 538846 ERROR (Thread-241) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 538945 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34155 [junit4] 2> 538947 INFO (zkConnectionManagerCallback-340-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 538951 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 538952 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 538958 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 538964 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 538964 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 538964 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.session Scavenging every 660000ms [junit4] 2> 538964 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ca792e9{/solr,null,AVAILABLE} [junit4] 2> 538965 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3558e1d9{SSL,[ssl, http/1.1]}{127.0.0.1:43550} [junit4] 2> 538965 INFO (jetty-launcher-337-thread-1) [ ] o.e.j.s.Server Started @549234ms [junit4] 2> 538966 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43550} [junit4] 2> 538966 ERROR (jetty-launcher-337-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 538966 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0 [junit4] 2> 538966 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 538966 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 538966 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-27T13:47:45.416Z [junit4] 2> 538969 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 538969 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 538970 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms [junit4] 2> 538970 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 538970 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 538970 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@9241e8f{/solr,null,AVAILABLE} [junit4] 2> 538970 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.session Scavenging every 660000ms [junit4] 2> 538970 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3c5f9b38{/solr,null,AVAILABLE} [junit4] 2> 538970 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7e983b33{SSL,[ssl, http/1.1]}{127.0.0.1:42351} [junit4] 2> 538970 INFO (jetty-launcher-337-thread-2) [ ] o.e.j.s.Server Started @549239ms [junit4] 2> 538971 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42351} [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2a34d87d{SSL,[ssl, http/1.1]}{127.0.0.1:40620} [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.e.j.s.Server Started @549239ms [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40620} [junit4] 2> 538971 ERROR (jetty-launcher-337-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 538971 ERROR (jetty-launcher-337-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 538971 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0 [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0 [junit4] 2> 538971 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 538971 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 538971 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-27T13:47:45.421Z [junit4] 2> 538971 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-27T13:47:45.421Z [junit4] 2> 539021 INFO (zkConnectionManagerCallback-342-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539022 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 539022 INFO (zkConnectionManagerCallback-346-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539022 INFO (zkConnectionManagerCallback-344-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539023 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 539025 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 539043 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x16137dfbd100001, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 539046 INFO (jetty-launcher-337-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34155/solr [junit4] 2> 539066 INFO (jetty-launcher-337-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34155/solr [junit4] 2> 539068 INFO (zkConnectionManagerCallback-350-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539069 INFO (jetty-launcher-337-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34155/solr [junit4] 2> 539070 INFO (zkConnectionManagerCallback-355-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539072 INFO (zkConnectionManagerCallback-358-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539090 INFO (zkConnectionManagerCallback-360-thread-1-processing-n:127.0.0.1:42351_solr) [n:127.0.0.1:42351_solr ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539092 INFO (zkConnectionManagerCallback-362-thread-1-processing-n:127.0.0.1:40620_solr) [n:127.0.0.1:40620_solr ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539111 INFO (zkConnectionManagerCallback-364-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 539189 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 539189 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 539208 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43550_solr [junit4] 2> 539209 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42351_solr [junit4] 2> 539210 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer Overseer (id=99422100868038665-127.0.0.1:43550_solr-n_0000000000) starting [junit4] 2> 539301 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer Starting to work on the main queue [junit4] 2> 539369 INFO (zkCallback-363-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 539369 INFO (zkCallback-359-thread-1-processing-n:127.0.0.1:42351_solr) [n:127.0.0.1:42351_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 539378 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 539390 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 539390 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:42351_solr] [junit4] 2> 539390 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1 [junit4] 2> 539390 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 539391 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43550_solr [junit4] 2> 539391 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"downnode", [junit4] 2> "node_name":"127.0.0.1:43550_solr"} current state version: 0 [junit4] 2> 539391 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:43550_solr [junit4] 2> 539422 INFO (zkCallback-359-thread-1-processing-n:127.0.0.1:42351_solr) [n:127.0.0.1:42351_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 539422 INFO (zkCallback-363-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 539424 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 539424 DEBUG (OverseerAutoScalingTriggerThread-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1 [junit4] 2> 539425 DEBUG (ScheduledTrigger-821-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 539626 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42351.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539709 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42351.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539711 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42351.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539712 INFO (jetty-launcher-337-thread-2) [n:127.0.0.1:42351_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_60CCFB5690A5FFE1-001/tempDir-001/node2/. [junit4] 2> 539730 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43550.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539749 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43550.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539766 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43550.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 539767 INFO (jetty-launcher-337-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_60CCFB5690A5FFE1-001/tempDir-001/node1/. [junit4] 2> 539820 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 539840 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 539857 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40620_solr [junit4] 2> 540068 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"downnode", [junit4] 2> "node_name":"127.0.0.1:40620_solr"} current state version: 0 [junit4] 2> 540068 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:40620_solr [junit4] 2> 540069 INFO (zkCallback-363-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 540094 INFO (zkCallback-359-thread-1-processing-n:127.0.0.1:42351_solr) [n:127.0.0.1:42351_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 540151 INFO (zkCallback-361-thread-1-processing-n:127.0.0.1:40620_solr) [n:127.0.0.1:40620_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 540247 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40620.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 540278 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40620.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 540278 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40620.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c1da4f0 [junit4] 2> 540297 INFO (jetty-launcher-337-thread-3) [n:127.0.0.1:40620_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_60CCFB5690A5FFE1-001/tempDir-001/node3/. [junit4] 2> 540491 INFO (zkConnectionManagerCallback-372-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 540511 DEBUG (ScheduledTrigger-821-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 540589 INFO (zkConnectionManagerCallback-376-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 540590 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3) [junit4] 2> 540590 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[60CCFB5690A5FFE1]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34155/solr ready [junit4] 2> 540872 INFO (qtp1406479260-1530) [n:127.0.0.1:40620_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&name=AutoscalingHistoryHandlerTest_collection&nrtReplicas=3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 540945 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection AutoscalingHistoryHandlerTest_collection [junit4] 2> 540946 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/protwords.txt [junit4] 2> 540948 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/managed-schema [junit4] 2> 540949 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/solrconfig.xml [junit4] 2> 540978 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/synonyms.txt [junit4] 2> 540979 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/stopwords.txt [junit4] 2> 540980 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/params.json to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/params.json [junit4] 2> 540981 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/contractions_it.txt [junit4] 2> 540994 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_no.txt [junit4] 2> 540995 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_hy.txt [junit4] 2> 540996 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stemdict_nl.txt [junit4] 2> 540996 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/contractions_ca.txt [junit4] 2> 540997 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_id.txt [junit4] 2> 541030 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ga.txt [junit4] 2> 541031 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_da.txt [junit4] 2> 541032 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_hi.txt [junit4] 2> 541033 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ja.txt [junit4] 2> 541033 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_pt.txt [junit4] 2> 541067 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_hu.txt [junit4] 2> 541068 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_el.txt [junit4] 2> 541068 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ru.txt [junit4] 2> 541069 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_tr.txt [junit4] 2> 541070 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ar.txt [junit4] 2> 541103 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/userdict_ja.txt [junit4] 2> 541104 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_it.txt [junit4] 2> 541105 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_eu.txt [junit4] 2> 541106 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_cz.txt [junit4] 2> 541107 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stoptags_ja.txt [junit4] 2> 541124 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/contractions_fr.txt [junit4] 2> 541125 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_fa.txt [junit4] 2> 541126 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_de.txt [junit4] 2> 541127 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/contractions_ga.txt [junit4] 2> 541140 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ca.txt [junit4] 2> 541141 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_nl.txt [junit4] 2> 541142 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_bg.txt [junit4] 2> 541142 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_es.txt [junit4] 2> 541143 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_en.txt [junit4] 2> 541160 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_lv.txt [junit4] 2> 541161 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_th.txt [junit4] 2> 541162 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_ro.txt [junit4] 2> 541163 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_fr.txt [junit4] 2> 541163 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_sv.txt [junit4] 2> 541176 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_fi.txt [junit4] 2> 541177 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/hyphenations_ga.txt [junit4] 2> 541178 INFO (OverseerThreadFactory-823-thread-1-processing-n:127.0.0.1:43550_solr) [n:127.0.0.1:43550_solr ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/lang/stopwords_gl.txt [junit4] 2> 541216 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "name":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "fromApi":"true", [junit4] 2> "replicationFactor":"3", [junit4] 2> "numShards":"1", [junit4] 2> "nrtReplicas":"3", [junit4] 2> "stateFormat":"2", [junit4] 2> "operation":"create"} current state version: 0 [junit4] 2> 541216 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.ClusterStateMutator building a new cName: AutoscalingHistoryHandlerTest_collection [junit4] 2> 541216 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/AutoscalingHistoryHandlerTest_collection/state.json [junit4] 2> 541332 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} current state version: 0 [junit4] 2> 541332 INFO (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 541333 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{}} [junit4] 2> 541333 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{"core_node3":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}}} [junit4] 2> 541334 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:40620/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} current state version: 0 [junit4] 2> 541334 INFO (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:40620/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 541379 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{"core_node3":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}}} [junit4] 2> 541379 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node5":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n2", [junit4] 2> "base_url":"https://127.0.0.1:40620/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}}} [junit4] 2> 541380 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 3, message = { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n4", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:43550/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} current state version: 0 [junit4] 2> 541380 INFO (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"AutoscalingHistoryHandlerTest_collection", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n4", [junit4] 2> "state":"down", [junit4] 2> "base_url":"https://127.0.0.1:43550/solr";, [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 541380 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node5":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n2", [junit4] 2> "base_url":"https://127.0.0.1:40620/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}}} [junit4] 2> 541380 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n1", [junit4] 2> "base_url":"https://127.0.0.1:42351/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node5":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n2", [junit4] 2> "base_url":"https://127.0.0.1:40620/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node6":{ [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n4", [junit4] 2> "base_url":"https://127.0.0.1:43550/solr";, [junit4] 2> "state":"down", [junit4] 2> "node_name":null, [junit4] 2> "type":"NRT"}}} [junit4] 2> 541496 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/AutoscalingHistoryHandlerTest_collection/state.json version: 0 [junit4] 2> 541540 DEBUG (ScheduledTrigger-821-thread-1) [n:127.0.0.1:43550_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 541744 INFO (qtp529960353-1520) [n:127.0.0.1:43550_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=AutoscalingHistoryHandlerTest_collection.AUTOCREATED&newCollection=true&name=AutoscalingHistoryHandlerTest_collection_shard1_replica_n4&action=CREATE&numShards=1&collection=AutoscalingHistoryHandlerTest_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 541745 INFO (qtp529960353-1520) [n:127.0.0.1:43550_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 541748 DEBUG (OverseerStateUpdate-99422100868038665-127.0.0.1:43550_solr-n_0000000000) [n:127.0.0.1:43550_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "core":"AutoscalingHistoryHandlerTest_collection_shard1_replica_n4", [junit4] 2> "core_node_name":"core_node6", [junit4] 2> "roles":null, [junit4] 2> "base_url":"https: [...truncated too long message...] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 7) Thread[id=24718, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) [junit4] > 8) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 9) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=25718, name=zkCallback-3077-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 2) Thread[id=24742, name=zkCallback-3077-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 3) Thread[id=24717, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[60CCFB5690A5FFE1]-SendThread(127.0.0.1:34454), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) [junit4] > 4) Thread[id=25672, name=zkCallback-3077-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 5) Thread[id=24719, name=zkConnectionManagerCallback-3078-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 6) Thread[id=26068, name=zkCallback-3077-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 7) Thread[id=25683, name=zkCallback-3077-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([60CCFB5690A5FFE1]:0) [junit4] Completed [389/770 (3!)] on J0 in 191.21s, 1 test, 3 errors <<< FAILURES! [...truncated 49194 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
