Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1333/

5 tests failed.
FAILED:  org.apache.solr.ltr.TestLTROnSolrCloud.testSimpleQuery

Error Message:
Path not found: /responseHeader/status

Stack Trace:
java.lang.RuntimeException: Path not found: /responseHeader/status
        at 
__randomizedtesting.SeedInfo.seed([1F7659E26DA23C8B:6A7B99FE7DB8FA5E]:0)
        at org.apache.solr.util.RestTestBase.assertJPut(RestTestBase.java:329)
        at org.apache.solr.util.RestTestBase.assertJPut(RestTestBase.java:271)
        at 
org.apache.solr.ltr.TestRerankBase.loadFeature(TestRerankBase.java:286)
        at 
org.apache.solr.ltr.TestLTROnSolrCloud.loadModelsAndFeatures(TestLTROnSolrCloud.java:184)
        at 
org.apache.solr.ltr.TestLTROnSolrCloud.setupSolrCluster(TestLTROnSolrCloud.java:138)
        at 
org.apache.solr.ltr.TestLTROnSolrCloud.setUp(TestLTROnSolrCloud.java:58)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth 
investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties 
were stopped - this is worth investigating!
        at 
__randomizedtesting.SeedInfo.seed([BAA083B17AB0FCEC:32F4BC6BD44C9114]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at 
org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
7 threads leaked from SUITE scope at 
org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:     1) 
Thread[id=5975, name=StoppableCommitThread-EventThread, state=WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    
2) Thread[id=5927, name=StoppableCommitThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)  
  3) Thread[id=5974, name=StoppableCommitThread-SendThread(127.0.0.1:57019), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]      
   at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    
4) Thread[id=6045, name=zkCallback-792-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    5) Thread[id=6044, 
name=zkCallback-792-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    6) Thread[id=5976, 
name=zkCallback-792-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    7) Thread[id=6046, 
name=zkCallback-792-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE 
scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 
   1) Thread[id=5975, name=StoppableCommitThread-EventThread, state=WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=5927, name=StoppableCommitThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
   3) Thread[id=5974, name=StoppableCommitThread-SendThread(127.0.0.1:57019), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=6045, name=zkCallback-792-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=6044, name=zkCallback-792-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=5976, name=zkCallback-792-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=6046, name=zkCallback-792-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([BAA083B17AB0FCEC]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=5974, name=StoppableCommitThread-SendThread(127.0.0.1:57019), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]      
   at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    2) 
Thread[id=6045, name=zkCallback-792-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    3) Thread[id=6044, 
name=zkCallback-792-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    4) Thread[id=5976, 
name=zkCallback-792-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)    5) Thread[id=6046, 
name=zkCallback-792-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=5974, name=StoppableCommitThread-SendThread(127.0.0.1:57019), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   2) Thread[id=6045, name=zkCallback-792-thread-3, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=6044, name=zkCallback-792-thread-2, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=5976, name=zkCallback-792-thread-1, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=6046, name=zkCallback-792-thread-4, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([BAA083B17AB0FCEC]:0)


FAILED:  
org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection

Error Message:
Expected to see nodes come back collection1 null Live Nodes: 
[127.0.0.1:47429_solr, 127.0.0.1:54391_solr, 127.0.0.1:50239_solr] Last 
available state: 
DocCollection(collection1//collections/collection1/state.json/21)={   
"pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{      
 "range":"80000000-ffffffff",       "state":"active",       "replicas":{        
 "core_node1":{           "core":"collection1_shard1_replica_n1",           
"base_url":"https://127.0.0.1:50239/solr";,           
"node_name":"127.0.0.1:50239_solr",           "state":"active",           
"type":"NRT"},         "core_node3":{           
"core":"collection1_shard1_replica_n2",           
"base_url":"https://127.0.0.1:56460/solr";,           
"node_name":"127.0.0.1:56460_solr",           "state":"down",           
"type":"NRT"},         "core_node4":{           
"core":"collection1_shard1_replica_n3",           
"base_url":"https://127.0.0.1:59716/solr";,           
"node_name":"127.0.0.1:59716_solr",           "state":"down",           
"type":"NRT"},         "core_node5":{           
"core":"collection1_shard1_replica_n4",           
"base_url":"https://127.0.0.1:57031/solr";,           
"node_name":"127.0.0.1:57031_solr",           "state":"down",           
"type":"NRT"},         "core_node6":{           
"core":"collection1_shard1_replica_n5",           
"base_url":"https://127.0.0.1:54391/solr";,           
"node_name":"127.0.0.1:54391_solr",           "state":"active",           
"type":"NRT",           "leader":"true"}}},     "shard2":{       
"range":"0-7fffffff",       "state":"active",       "replicas":{"core_node2":{  
         "core":"collection1_shard2_replica_n1",           
"base_url":"https://127.0.0.1:47429/solr";,           
"node_name":"127.0.0.1:47429_solr",           "state":"active",           
"type":"NRT",           "leader":"true"}}}},   "router":{"name":"compositeId"}, 
  "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   
"tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected to see nodes come back collection1
null
Live Nodes: [127.0.0.1:47429_solr, 127.0.0.1:54391_solr, 127.0.0.1:50239_solr]
Last available state: 
DocCollection(collection1//collections/collection1/state.json/21)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"collection1_shard1_replica_n1",
          "base_url":"https://127.0.0.1:50239/solr";,
          "node_name":"127.0.0.1:50239_solr",
          "state":"active",
          "type":"NRT"},
        "core_node3":{
          "core":"collection1_shard1_replica_n2",
          "base_url":"https://127.0.0.1:56460/solr";,
          "node_name":"127.0.0.1:56460_solr",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "base_url":"https://127.0.0.1:59716/solr";,
          "node_name":"127.0.0.1:59716_solr",
          "state":"down",
          "type":"NRT"},
        "core_node5":{
          "core":"collection1_shard1_replica_n4",
          "base_url":"https://127.0.0.1:57031/solr";,
          "node_name":"127.0.0.1:57031_solr",
          "state":"down",
          "type":"NRT"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "base_url":"https://127.0.0.1:54391/solr";,
          "node_name":"127.0.0.1:54391_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"collection1_shard2_replica_n1",
          "base_url":"https://127.0.0.1:47429/solr";,
          "node_name":"127.0.0.1:47429_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([BAA083B17AB0FCEC:E4944EFF39CF600B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
        at 
org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection(LeaderElectionIntegrationTest.java:109)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12001 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderElectionIntegrationTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/init-core-data-001
   [junit4]   2> 975731 INFO  
(SUITE-LeaderElectionIntegrationTest-seed#[BAA083B17AB0FCEC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 975733 INFO  
(SUITE-LeaderElectionIntegrationTest-seed#[BAA083B17AB0FCEC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 975736 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testSimpleSliceLeaderElection
   [junit4]   2> 975737 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 6 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001
   [junit4]   2> 975737 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 975737 INFO  (Thread-1595) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 975738 INFO  (Thread-1595) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 975751 ERROR (Thread-1595) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 975837 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.c.ZkTestServer start zk server on port:35262
   [junit4]   2> 975849 INFO  (jetty-launcher-676-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975853 INFO  (jetty-launcher-676-thread-3) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975857 INFO  (jetty-launcher-676-thread-4) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975859 INFO  (jetty-launcher-676-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975871 INFO  (jetty-launcher-676-thread-5) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975872 INFO  (jetty-launcher-676-thread-6) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 975873 INFO  (jetty-launcher-676-thread-4) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@45568228{/solr,null,AVAILABLE}
   [junit4]   2> 975904 INFO  (jetty-launcher-676-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3f3308c3{/solr,null,AVAILABLE}
   [junit4]   2> 975905 INFO  (jetty-launcher-676-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@6cdfd775{SSL,[ssl, 
http/1.1]}{127.0.0.1:57031}
   [junit4]   2> 975905 INFO  (jetty-launcher-676-thread-1) [    ] 
o.e.j.s.Server Started @982821ms
   [junit4]   2> 975905 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=57031}
   [junit4]   2> 975905 ERROR (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975918 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975918 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975918 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975918 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.452Z
   [junit4]   2> 975918 INFO  (jetty-launcher-676-thread-4) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@6197103a{SSL,[ssl, 
http/1.1]}{127.0.0.1:59716}
   [junit4]   2> 975919 INFO  (jetty-launcher-676-thread-4) [    ] 
o.e.j.s.Server Started @982835ms
   [junit4]   2> 975919 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=59716}
   [junit4]   2> 975920 ERROR (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975920 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975920 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975920 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975920 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.454Z
   [junit4]   2> 975923 INFO  (jetty-launcher-676-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6208e7d7{/solr,null,AVAILABLE}
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@5d55e7a5{SSL,[ssl, 
http/1.1]}{127.0.0.1:56460}
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.e.j.s.Server Started @982840ms
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=56460}
   [junit4]   2> 975924 ERROR (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975924 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.458Z
   [junit4]   2> 975936 INFO  (jetty-launcher-676-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@7530f55a{/solr,null,AVAILABLE}
   [junit4]   2> 975953 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 975954 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 975961 INFO  (jetty-launcher-676-thread-6) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1f718cfc{/solr,null,AVAILABLE}
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@53d8f981{SSL,[ssl, 
http/1.1]}{127.0.0.1:47429}
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.e.j.s.Server Started @982878ms
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=47429}
   [junit4]   2> 975962 ERROR (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975962 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.496Z
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@58bcd953{SSL,[ssl, 
http/1.1]}{127.0.0.1:50239}
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.e.j.s.Server Started @982884ms
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=50239}
   [junit4]   2> 975968 ERROR (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975968 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.502Z
   [junit4]   2> 975977 INFO  (jetty-launcher-676-thread-5) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1387274a{/solr,null,AVAILABLE}
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@657054d7{SSL,[ssl, 
http/1.1]}{127.0.0.1:54391}
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.e.j.s.Server Started @982894ms
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=54391}
   [junit4]   2> 975978 ERROR (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 975978 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-22T05:55:47.512Z
   [junit4]   2> 975988 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 975989 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 975992 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 975994 INFO  (jetty-launcher-676-thread-4) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976002 INFO  (jetty-launcher-676-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976007 INFO  (jetty-launcher-676-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976013 INFO  (jetty-launcher-676-thread-6) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976026 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15cce5eefc90008, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 976031 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 976032 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15cce5eefc90009, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 976047 INFO  (jetty-launcher-676-thread-5) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976068 INFO  (jetty-launcher-676-thread-3) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35262/solr
   [junit4]   2> 976091 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976095 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976091 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976105 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:56460_solr
   [junit4]   2> 976107 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:47429_solr
   [junit4]   2> 976107 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:59716_solr
   [junit4]   2> 976108 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.Overseer Overseer 
(id=98180199040483340-127.0.0.1:56460_solr-n_0000000000) starting
   [junit4]   2> 976121 INFO  
(zkCallback-698-thread-1-processing-n:127.0.0.1:59716_solr) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976121 INFO  
(zkCallback-699-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976141 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976146 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976147 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976148 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 976149 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976150 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976151 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 976153 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:50239_solr
   [junit4]   2> 976153 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:54391_solr
   [junit4]   2> 976154 INFO  
(zkCallback-698-thread-1-processing-n:127.0.0.1:59716_solr) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 976154 INFO  
(zkCallback-699-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 976154 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 976155 INFO  
(zkCallback-699-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 976155 INFO  
(zkCallback-698-thread-1-processing-n:127.0.0.1:59716_solr) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 976155 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 976191 INFO  
(zkCallback-709-thread-1-processing-n:127.0.0.1:54391_solr) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (4)
   [junit4]   2> 976191 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (4)
   [junit4]   2> 976200 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:57031_solr
   [junit4]   2> 976202 INFO  
(zkCallback-700-thread-1-processing-n:127.0.0.1:57031_solr) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (4)
   [junit4]   2> 976220 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:56460_solr
   [junit4]   2> 976220 INFO  
(zkCallback-699-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (5)
   [junit4]   2> 976220 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (5)
   [junit4]   2> 976220 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (5)
   [junit4]   2> 976220 INFO  
(zkCallback-709-thread-1-processing-n:127.0.0.1:54391_solr) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (5)
   [junit4]   2> 976221 INFO  
(zkCallback-698-thread-1-processing-n:127.0.0.1:59716_solr) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (5)
   [junit4]   2> 976221 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (5) -> (6)
   [junit4]   2> 976221 INFO  
(zkCallback-699-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (5) -> (6)
   [junit4]   2> 976221 INFO  
(zkCallback-709-thread-1-processing-n:127.0.0.1:54391_solr) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (5) -> (6)
   [junit4]   2> 976222 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (5) -> (6)
   [junit4]   2> 976222 INFO  
(zkCallback-698-thread-1-processing-n:127.0.0.1:59716_solr) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (5) -> (6)
   [junit4]   2> 976228 INFO  
(zkCallback-700-thread-1-processing-n:127.0.0.1:57031_solr) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (6)
   [junit4]   2> 976321 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47429.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976324 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_59716.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976336 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_59716.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976336 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_59716.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976337 INFO  (jetty-launcher-676-thread-4) 
[n:127.0.0.1:59716_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node4/.
   [junit4]   2> 976377 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47429.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976378 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47429.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976379 INFO  (jetty-launcher-676-thread-6) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node6/.
   [junit4]   2> 976394 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_54391.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976412 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_54391.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976427 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_54391.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976428 INFO  (jetty-launcher-676-thread-5) 
[n:127.0.0.1:54391_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node5/.
   [junit4]   2> 976435 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50239.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976465 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50239.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976465 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50239.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976468 INFO  (jetty-launcher-676-thread-3) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node3/.
   [junit4]   2> 976470 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_56460.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976472 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57031.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976484 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_56460.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976485 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_56460.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976486 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57031.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976486 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_57031.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 976491 INFO  (jetty-launcher-676-thread-2) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node2/.
   [junit4]   2> 976492 INFO  (jetty-launcher-676-thread-1) 
[n:127.0.0.1:57031_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node1/.
   [junit4]   2> 976688 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (6)
   [junit4]   2> 976690 INFO  
(TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[BAA083B17AB0FCEC])
 [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 
127.0.0.1:35262/solr ready
   [junit4]   2> 976745 INFO  (qtp335529671-5393) [n:127.0.0.1:47429_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf&maxShardsPerNode=1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 976749 INFO  
(OverseerThreadFactory-2639-thread-1-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
collection1
   [junit4]   2> 976865 INFO  
(OverseerStateUpdate-98180199040483340-127.0.0.1:56460_solr-n_0000000000) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:50239/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 976867 INFO  
(OverseerStateUpdate-98180199040483340-127.0.0.1:56460_solr-n_0000000000) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47429/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 977075 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 977075 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 977105 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 977106 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 977217 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 977217 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 977219 INFO  
(zkCallback-711-thread-2-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 977224 INFO  
(zkCallback-703-thread-2-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978094 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 978110 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=minimal
   [junit4]   2> 978114 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 978114 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 978115 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_47429.solr.core.collection1.shard2.replica_n1' (registry 
'solr.core.collection1.shard2.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 978115 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 978115 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node6/collection1_shard2_replica_n1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node6/./collection1_shard2_replica_n1/data/]
   [junit4]   2> 978120 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 978128 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 978130 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 978130 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 978131 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50239.solr.core.collection1.shard1.replica_n1' (registry 
'solr.core.collection1.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 978131 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 978131 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node3/collection1_shard1_replica_n1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node3/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 978174 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 978174 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 978175 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 978175 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 978177 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1d97a78d[collection1_shard2_replica_n1] main]
   [junit4]   2> 978180 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 978181 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 978182 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 978182 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 978182 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 978182 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1570883187209404416
   [junit4]   2> 978183 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 978183 INFO  
(searcherExecutor-2652-thread-1-processing-n:127.0.0.1:47429_solr 
x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node2) 
[n:127.0.0.1:47429_solr c:collection1 s:shard2 r:core_node2 
x:collection1_shard2_replica_n1] o.a.s.c.SolrCore 
[collection1_shard2_replica_n1] Registered new searcher 
Searcher@1d97a78d[collection1_shard2_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 978183 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 978185 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@634aab4d[collection1_shard1_replica_n1] main]
   [junit4]   2> 978186 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 978187 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 978187 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 978188 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1570883187215695872
   [junit4]   2> 978188 INFO  
(searcherExecutor-2653-thread-1-processing-n:127.0.0.1:50239_solr 
x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node1) 
[n:127.0.0.1:50239_solr c:collection1 s:shard1 r:core_node1 
x:collection1_shard1_replica_n1] o.a.s.c.SolrCore 
[collection1_shard1_replica_n1] Registered new searcher 
Searcher@634aab4d[collection1_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 978192 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 978192 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 978192 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 978197 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 978197 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 978197 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 978199 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:47429/solr/collection1_shard2_replica_n1/ shard2
   [junit4]   2> 978202 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:50239/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 978305 INFO  
(zkCallback-703-thread-2-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978305 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978305 INFO  
(zkCallback-711-thread-2-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978306 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978350 WARN  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.c.ZkController Skipping recovery according to sys prop 
solrcloud.skip.autorecovery
   [junit4]   2> 978353 WARN  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkController Skipping recovery according to sys prop 
solrcloud.skip.autorecovery
   [junit4]   2> 978359 INFO  (qtp335529671-5380) [n:127.0.0.1:47429_solr 
c:collection1 s:shard2 r:core_node2 x:collection1_shard2_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1284
   [junit4]   2> 978359 INFO  (qtp203567805-5373) [n:127.0.0.1:50239_solr 
c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1253
   [junit4]   2> 978365 INFO  (qtp335529671-5393) [n:127.0.0.1:47429_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 978463 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978463 INFO  
(zkCallback-711-thread-2-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978463 INFO  
(zkCallback-703-thread-2-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978463 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 978750 INFO  
(OverseerCollectionConfigSetProcessor-98180199040483340-127.0.0.1:56460_solr-n_0000000000)
 [n:127.0.0.1:56460_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 979366 INFO  (qtp335529671-5393) [n:127.0.0.1:47429_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf&maxShardsPerNode=1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2}
 status=0 QTime=2620
   [junit4]   2> 979367 INFO  (qtp335529671-5401) [n:127.0.0.1:47429_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params 
action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 979369 INFO  
(OverseerThreadFactory-2639-thread-2-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.AddReplicaCmd addReplica() : {
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"addreplica"}
   [junit4]   2> 979371 INFO  
(OverseerThreadFactory-2639-thread-2-processing-n:127.0.0.1:56460_solr) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.AddReplicaCmd Node Identified 
127.0.0.1:56460_solr for creating new replica
   [junit4]   2> 979372 INFO  
(OverseerStateUpdate-98180199040483340-127.0.0.1:56460_solr-n_0000000000) 
[n:127.0.0.1:56460_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:56460/solr";,
   [junit4]   2>   "node_name":"127.0.0.1:56460_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 979474 INFO  
(zkCallback-711-thread-2-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 979474 INFO  
(zkCallback-703-thread-2-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 979474 INFO  
(zkCallback-711-thread-1-processing-n:127.0.0.1:50239_solr) 
[n:127.0.0.1:50239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 979474 INFO  
(zkCallback-703-thread-1-processing-n:127.0.0.1:47429_solr) 
[n:127.0.0.1:47429_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [6])
   [junit4]   2> 979603 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&name=collection1_shard1_replica_n2&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 979603 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 979616 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 979624 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 979627 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 979627 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n2' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 979627 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_56460.solr.core.collection1.shard1.replica_n2' (registry 
'solr.core.collection1.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@77a7ea62
   [junit4]   2> 979628 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 979628 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node2/collection1_shard1_replica_n2],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionIntegrationTest_BAA083B17AB0FCEC-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 979685 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 979685 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 979688 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 979688 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 979689 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@21bdf941[collection1_shard1_replica_n2] main]
   [junit4]   2> 979691 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 979692 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 979693 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 979693 INFO  (qtp544983802-5385) [n:127.0.0.1:56460_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1570883188793802752
   [junit4]   2> 979695 INFO  
(searcherExecutor-2662-thread-1-processing-n:127.0.0.1:56460_solr 
x:collection1_shard1_replica_n2 s:shard1 c:collection1 r:core_node3) 
[n:127.0.0.1:56460_solr c:collection1 s:shard1 r:core_node3 x:collection1_

[...truncated too long message...]

g=1375330808
   [junit4]   2> 25536 WARN  
(zkCallback-45-thread-3-processing-n:127.0.0.1:44042_solr) 
[n:127.0.0.1:44042_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 25548 INFO  
(zkCallback-40-thread-1-processing-n:127.0.0.1:51305_solr) 
[n:127.0.0.1:51305_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 25548 INFO  
(zkCallback-47-thread-2-processing-n:127.0.0.1:32899_solr) 
[n:127.0.0.1:32899_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 25548 INFO  
(zkCallback-43-thread-3-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 25563 INFO  (jetty-closer-6-thread-6) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1e3be348{/solr,null,UNAVAILABLE}
   [junit4]   2> 25563 INFO  
(zkCallback-43-thread-3-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 25564 INFO  
(zkCallback-43-thread-1-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 25567 INFO  
(zkCallback-40-thread-1-processing-n:127.0.0.1:51305_solr) 
[n:127.0.0.1:51305_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 25568 INFO  
(zkCallback-40-thread-2-processing-n:127.0.0.1:51305_solr) 
[n:127.0.0.1:51305_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 25569 INFO  (coreCloseExecutor-111-thread-1) 
[n:127.0.0.1:33511_solr c:collection1 s:shard2 r:core_node2 
x:collection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.collection1.shard2.leader, tag=100435185
   [junit4]   2> 25578 INFO  
(zkCallback-43-thread-2-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 25579 INFO  
(zkCallback-40-thread-4-processing-n:127.0.0.1:51305_solr) 
[n:127.0.0.1:51305_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 25583 INFO  (jetty-closer-6-thread-4) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@20c099f4{/solr,null,UNAVAILABLE}
   [junit4]   2> 25583 INFO  (coreCloseExecutor-110-thread-1) 
[n:127.0.0.1:51305_solr c:collection1 s:shard2 r:core_node4 
x:collection1_shard2_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.collection1.shard2.leader, tag=927063262
   [junit4]   2> 25589 INFO  
(zkCallback-43-thread-2-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 25589 INFO  (jetty-closer-6-thread-3) [    ] o.a.s.c.Overseer 
Overseer (id=98180300533006355-127.0.0.1:33511_solr-n_0000000000) closing
   [junit4]   2> 25590 INFO  
(OverseerStateUpdate-98180300533006355-127.0.0.1:33511_solr-n_0000000000) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:33511_solr
   [junit4]   2> 25606 WARN  
(zkCallback-43-thread-2-processing-n:127.0.0.1:33511_solr) 
[n:127.0.0.1:33511_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 25607 INFO  (jetty-closer-6-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@70f78625{/solr,null,UNAVAILABLE}
   [junit4]   2> 26906 WARN  
(zkCallback-42-thread-1-processing-n:127.0.0.1:42771_solr) 
[n:127.0.0.1:42771_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 26907 INFO  (jetty-closer-6-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@315f1f91{/solr,null,UNAVAILABLE}
   [junit4]   2> 26911 WARN  
(zkCallback-44-thread-1-processing-n:127.0.0.1:48940_solr) 
[n:127.0.0.1:48940_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 26912 INFO  (jetty-closer-6-thread-5) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@14f02f29{/solr,null,UNAVAILABLE}
   [junit4]   2> 27089 WARN  
(zkCallback-40-thread-4-processing-n:127.0.0.1:51305_solr) 
[n:127.0.0.1:51305_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 27089 INFO  (jetty-closer-6-thread-7) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@4088d023{/solr,null,UNAVAILABLE}
   [junit4]   2> 27091 ERROR 
(TEST-TestLTROnSolrCloud.testSimpleQuery-seed#[1F7659E26DA23C8B]) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 27093 INFO  
(TEST-TestLTROnSolrCloud.testSimpleQuery-seed#[1F7659E26DA23C8B]) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:46208 46208
   [junit4]   2> 27227 INFO  (Thread-12) [    ] o.a.s.c.ZkTestServer connecting 
to 127.0.0.1:46208 46208
   [junit4]   2> 27228 WARN  (Thread-12) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        10      /solr/aliases.json
   [junit4]   2>        9       /solr/security.json
   [junit4]   2>        4       /solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        33      /solr/collections/collection1/state.json
   [junit4]   2>        10      /solr/clusterprops.json
   [junit4]   2>        10      /solr/clusterstate.json
   [junit4]   2>        3       
/solr/overseer_elect/election/98180300533006355-127.0.0.1:33511_solr-n_0000000000
   [junit4]   2>        2       
/solr/overseer_elect/election/98180300533006360-127.0.0.1:32899_solr-n_0000000002
   [junit4]   2>        2       
/solr/overseer_elect/election/98180300533006358-127.0.0.1:43528_solr-n_0000000003
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        10      /solr/live_nodes
   [junit4]   2>        10      /solr/collections
   [junit4]   2> 
   [junit4]   2> 27229 INFO  
(TEST-TestLTROnSolrCloud.testSimpleQuery-seed#[1F7659E26DA23C8B]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testSimpleQuery
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLTROnSolrCloud 
-Dtests.method=testSimpleQuery -Dtests.seed=1F7659E26DA23C8B 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=fr-CH -Dtests.timezone=Singapore -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   16.0s J0 | TestLTROnSolrCloud.testSimpleQuery <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Path not found: 
/responseHeader/status
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([1F7659E26DA23C8B:6A7B99FE7DB8FA5E]:0)
   [junit4]    >        at 
org.apache.solr.util.RestTestBase.assertJPut(RestTestBase.java:329)
   [junit4]    >        at 
org.apache.solr.util.RestTestBase.assertJPut(RestTestBase.java:271)
   [junit4]    >        at 
org.apache.solr.ltr.TestRerankBase.loadFeature(TestRerankBase.java:286)
   [junit4]    >        at 
org.apache.solr.ltr.TestLTROnSolrCloud.loadModelsAndFeatures(TestLTROnSolrCloud.java:184)
   [junit4]    >        at 
org.apache.solr.ltr.TestLTROnSolrCloud.setupSolrCluster(TestLTROnSolrCloud.java:138)
   [junit4]    >        at 
org.apache.solr.ltr.TestLTROnSolrCloud.setUp(TestLTROnSolrCloud.java:58)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-ltr/test/J0/temp/solr.ltr.TestLTROnSolrCloud_1F7659E26DA23C8B-001
   [junit4]   2> Jun 22, 2017 6:21:51 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{_version_=FST50, popularity=PostingsFormat(name=LuceneVarGapFixedInterval), 
description=PostingsFormat(name=Memory), 
id=PostingsFormat(name=LuceneVarGapFixedInterval), 
text=PostingsFormat(name=Asserting), title=PostingsFormat(name=Memory)}, 
docValues:{}, maxPointsInLeafNode=1897, maxMBSortInHeap=6.41759853689495, 
sim=RandomSimilarity(queryNorm=false): {}, locale=fr-CH, timezone=Singapore
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_131 (64-bit)/cpus=4,threads=1,free=283060400,total=343408640
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestMultipleAdditiveTreesModel, TestLTROnSolrCloud]
   [junit4] Completed [20/33 (1!)] on J0 in 16.73s, 1 test, 1 error <<< 
FAILURES!

[...truncated 36843 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to