Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.5/29/

5 tests failed.
FAILED:  
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at 
__randomizedtesting.SeedInfo.seed([90B6D555A228D407:1B910684E32E7F83]:0)
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:433)
        at sun.nio.ch.Net.bind(Net.java:425)
        at 
sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at 
org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at 
org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:307)
        at 
org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at 
org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:231)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at org.eclipse.jetty.server.Server.doStart(Server.java:385)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:565)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:504)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:472)
        at 
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
10 threads leaked from SUITE scope at 
org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=74485, 
name=qtp1190481202-74485, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=74486, 
name=qtp1190481202-74486, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=74488, 
name=qtp1190481202-74488, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=74491, 
name=qtp1190481202-74491, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=74489, 
name=qtp1190481202-74489-acceptor-0@16cd57d3-ServerConnector@42cad134{HTTP/1.1,[http/1.1,
 h2c]}{127.0.0.1:38614}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=74487, 
name=qtp1190481202-74487, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=74490, 
name=qtp1190481202-74490, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=74492, 
name=qtp1190481202-74492, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=74493, 
name=Session-HouseKeeper-d936931-1, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)   10) Thread[id=74581, 
name=Connector-Scheduler-42cad134-1, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from 
SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 
   1) Thread[id=74485, name=qtp1190481202-74485, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=74486, name=qtp1190481202-74486, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=74488, name=qtp1190481202-74488, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=74491, name=qtp1190481202-74491, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=74489, 
name=qtp1190481202-74489-acceptor-0@16cd57d3-ServerConnector@42cad134{HTTP/1.1,[http/1.1,
 h2c]}{127.0.0.1:38614}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=74487, name=qtp1190481202-74487, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=74490, name=qtp1190481202-74490, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=74492, name=qtp1190481202-74492, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=74493, name=Session-HouseKeeper-d936931-1, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=74581, name=Connector-Scheduler-42cad134-1, 
state=TIMED_WAITING, group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([90B6D555A228D407]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=74485, name=qtp1190481202-74485, state=RUNNABLE, 
group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=74486, 
name=qtp1190481202-74486, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=74488, 
name=qtp1190481202-74488, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=74491, 
name=qtp1190481202-74491, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=74487, 
name=qtp1190481202-74487, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
         at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=74489, 
name=qtp1190481202-74489, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=74490, 
name=qtp1190481202-74490, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=74492, 
name=qtp1190481202-74492, state=TIMED_WAITING, group=TGRP-ShardSplitTest]       
  at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=74493, 
name=Session-HouseKeeper-d936931-1, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)   10) Thread[id=74581, 
name=Connector-Scheduler-42cad134-1, state=WAITING, group=TGRP-ShardSplitTest]  
       at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=74485, name=qtp1190481202-74485, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=74486, name=qtp1190481202-74486, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=74488, name=qtp1190481202-74488, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=74491, name=qtp1190481202-74491, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=74487, name=qtp1190481202-74487, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at 
org.eclipse.jetty.io.ManagedSelector$$Lambda$53/2096923016.run(Unknown Source)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=74489, name=qtp1190481202-74489, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=74490, name=qtp1190481202-74490, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=74492, name=qtp1190481202-74492, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=74493, name=Session-HouseKeeper-d936931-1, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=74581, name=Connector-Scheduler-42cad134-1, state=WAITING, 
group=TGRP-ShardSplitTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([90B6D555A228D407]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
There are still nodes recoverying - waited for 60 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 60 
seconds
        at 
__randomizedtesting.SeedInfo.seed([90B6D555A228D407:18E2EA8F0CD4B9FF]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:196)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1061)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1618)
        at 
org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:391)
        at 
org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:102)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test

Error Message:
expected:<5> but was:<4>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<4>
        at 
__randomizedtesting.SeedInfo.seed([90B6D555A228D407:18E2EA8F0CD4B9FF]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at org.junit.Assert.failNotEquals(Assert.java:834)
        at org.junit.Assert.assertEquals(Assert.java:645)
        at org.junit.Assert.assertEquals(Assert.java:631)
        at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:150)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13926 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsSyncSliceTest
   [junit4]   2> 1489377 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to 
test-framework derived value of 
'/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 1489378 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Created dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001/data-dir-79-001
   [junit4]   2> 1489378 WARN  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=55 numCloses=55
   [junit4]   2> 1489378 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1489380 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1489380 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1489380 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /t_zi/
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1491113 WARN  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1491151 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 
363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 1491155 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1491155 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1491155 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1491158 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@502883c{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1491382 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@71f6bc91{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/jetty-lucene2-us-west_apache_org-37466-hadoop-hdfs-3_2_0-tests_jar-_-any-8617599631465854979.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 1491382 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@53ac726e{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:37466}
   [junit4]   2> 1491382 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
Started @1491419ms
   [junit4]   2> 1492251 WARN  (StorageLocationChecker thread 0) [     ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   2> 1492334 WARN  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1492342 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 
363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 1492342 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1492342 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1492342 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1492343 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@7336bfca{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1492469 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@175211dc{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-46465-hadoop-hdfs-3_2_0-tests_jar-_-any-5120735235229953175.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 1492469 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@63624118{HTTP/1.1,[http/1.1]}{localhost:46465}
   [junit4]   2> 1492469 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
Started @1492505ms
   [junit4]   2> 1493027 WARN  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1493028 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 
363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 1493029 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1493029 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1493029 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1493029 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6d2a7ef{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1493159 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@54aebdc7{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-35152-hadoop-hdfs-3_2_0-tests_jar-_-any-7711663327791994505.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 1493159 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@66aedffd{HTTP/1.1,[http/1.1]}{localhost:35152}
   [junit4]   2> 1493159 INFO  
(SUITE-HdfsSyncSliceTest-seed#[90B6D555A228D407]-worker) [     ] o.e.j.s.Server 
Started @1493196ms
   [junit4]   2> 1493979 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x91f16eb8eac3d531: Processing first 
storage report for DS-7df41f55-831b-4113-b48a-6b74435b4974 from datanode 
7e3dfec0-5863-423a-ba07-412146c0baca
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x91f16eb8eac3d531: from storage 
DS-7df41f55-831b-4113-b48a-6b74435b4974 node 
DatanodeRegistration(127.0.0.1:35291, 
datanodeUuid=7e3dfec0-5863-423a-ba07-412146c0baca, infoPort=35549, 
infoSecurePort=0, ipcPort=40005, 
storageInfo=lv=-57;cid=testClusterID;nsid=1344709808;c=1584474976967), blocks: 
0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x133cefa656f5fb11: Processing first 
storage report for DS-91841bd0-c18f-4cb7-848d-89c694791cf7 from datanode 
e17322da-fd3c-4bb2-a3bd-364667d355c1
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x133cefa656f5fb11: from storage 
DS-91841bd0-c18f-4cb7-848d-89c694791cf7 node 
DatanodeRegistration(127.0.0.1:33394, 
datanodeUuid=e17322da-fd3c-4bb2-a3bd-364667d355c1, infoPort=40020, 
infoSecurePort=0, ipcPort=39392, 
storageInfo=lv=-57;cid=testClusterID;nsid=1344709808;c=1584474976967), blocks: 
0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x91f16eb8eac3d531: Processing first 
storage report for DS-a4af24f1-e726-4683-80a9-a2b9aa231530 from datanode 
7e3dfec0-5863-423a-ba07-412146c0baca
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x91f16eb8eac3d531: from storage 
DS-a4af24f1-e726-4683-80a9-a2b9aa231530 node 
DatanodeRegistration(127.0.0.1:35291, 
datanodeUuid=7e3dfec0-5863-423a-ba07-412146c0baca, infoPort=35549, 
infoSecurePort=0, ipcPort=40005, 
storageInfo=lv=-57;cid=testClusterID;nsid=1344709808;c=1584474976967), blocks: 
0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x133cefa656f5fb11: Processing first 
storage report for DS-cfafcbc2-760a-4bbc-a502-c1ca99195b9d from datanode 
e17322da-fd3c-4bb2-a3bd-364667d355c1
   [junit4]   2> 1493982 INFO  (Block report processor) [     ] 
BlockStateChange BLOCK* processReport 0x133cefa656f5fb11: from storage 
DS-cfafcbc2-760a-4bbc-a502-c1ca99195b9d node 
DatanodeRegistration(127.0.0.1:33394, 
datanodeUuid=e17322da-fd3c-4bb2-a3bd-364667d355c1, infoPort=40020, 
infoSecurePort=0, ipcPort=39392, 
storageInfo=lv=-57;cid=testClusterID;nsid=1344709808;c=1584474976967), blocks: 
0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1494169 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1494170 INFO  (ZkTestServer Run Thread) [     ] 
o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1494170 INFO  (ZkTestServer Run Thread) [     ] 
o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1494270 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer start zk server on port:45932
   [junit4]   2> 1494270 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45932
   [junit4]   2> 1494270 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45932
   [junit4]   2> 1494270 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1 45932
   [junit4]   2> 1494272 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494276 INFO  (zkConnectionManagerCallback-5383-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1494276 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1494280 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494281 INFO  (zkConnectionManagerCallback-5385-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1494281 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1494284 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 1494286 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 1494287 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1494289 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 1494290 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 1494291 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 1494292 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1494293 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1494294 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1494295 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1494296 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkTestServer put 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 1494297 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly 
asked otherwise
   [junit4]   2> 1494447 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, 
break even is size 23
   [junit4]   2> 1494447 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1494447 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 1494447 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] o.e.j.s.Server 
jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 
363d5f2df3a8a28de40604320230664b9c793c16; jvm 1.8.0_191-b12
   [junit4]   2> 1494448 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] o.e.j.s.session 
DefaultSessionIdManager workerName=node0
   [junit4]   2> 1494448 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] o.e.j.s.session 
No SessionScavenger set, using defaults
   [junit4]   2> 1494448 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] o.e.j.s.session 
node0 Scavenging every 600000ms
   [junit4]   2> 1494450 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@589fc190{/t_zi,null,AVAILABLE}
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.s.AbstractConnector Started ServerConnector@687ae631{HTTP/1.1,[http/1.1, 
h2c]}{127.0.0.1:43420}
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] o.e.j.s.Server 
Started @1494487ms
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=hdfs://lucene2-us-west.apache.org:42008/hdfs__lucene2-us-west.apache.org_42008__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-8.5_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001_tempDir-002_control_data,
 hostContext=/t_zi, hostPort=43420, 
coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001/control-001/cores}
   [junit4]   2> 1494451 ERROR 
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.5.0
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr
   [junit4]   2> 1494451 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2020-03-17T19:56:20.836Z
   [junit4]   2> 1494452 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494454 INFO  (zkConnectionManagerCallback-5387-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1494454 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1494555 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1494556 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001/control-001/solr.xml
   [junit4]   2> 1494560 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay 
is ignored
   [junit4]   2> 1494560 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.SolrXmlConfig Configuration parameter 
autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1494561 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1cc48781, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1496233 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: 
WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 1496234 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.u.s.S.config Trusting all certificates configured for 
Client@53a33153[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1496234 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
Client@53a33153[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1496237 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.u.s.S.config Trusting all certificates configured for 
Client@243ed19e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1496237 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
Client@243ed19e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1496238 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45932/solr
   [junit4]   2> 1496240 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496241 INFO  (zkConnectionManagerCallback-5394-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1496241 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496344 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496345 INFO  (zkConnectionManagerCallback-5396-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1496345 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496432 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.OverseerElectionContext I am going to be the leader 
127.0.0.1:43420_t_zi
   [junit4]   2> 1496432 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.Overseer Overseer 
(id=72467227863810052-127.0.0.1:43420_t_zi-n_0000000000) starting
   [junit4]   2> 1496438 INFO  
(OverseerStateUpdate-72467227863810052-127.0.0.1:43420_t_zi-n_0000000000) 
[n:127.0.0.1:43420_t_zi     ] o.a.s.c.Overseer Starting to work on the main 
queue : 127.0.0.1:43420_t_zi
   [junit4]   2> 1496439 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:43420_t_zi
   [junit4]   2> 1496440 INFO  
(OverseerStateUpdate-72467227863810052-127.0.0.1:43420_t_zi-n_0000000000) 
[n:127.0.0.1:43420_t_zi     ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1496443 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1496444 WARN  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.CoreContainer Not all security plugins configured!  
authentication=disabled authorization=disabled.  Solr is only as secure as you 
make it. Consider configuring authentication/authorization before exposing Solr 
to users internal or external.  See https://s.apache.org/solrsecurity for more 
info
   [junit4]   2> 1496464 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics 
history in memory.
   [junit4]   2> 1496504 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 
'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cc48781
   [junit4]   2> 1496518 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 
'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cc48781
   [junit4]   2> 1496518 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 
'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cc48781
   [junit4]   2> 1496520 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [n:127.0.0.1:43420_t_zi   
  ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001/control-001/cores
   [junit4]   2> 1496547 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496548 INFO  (zkConnectionManagerCallback-5405-thread-1) [    
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1496548 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496550 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1496551 INFO  
(TEST-HdfsSyncSliceTest.test-seed#[90B6D555A228D407]) [     ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45932/solr ready
   [junit4]   2> 1496553 INFO  (qtp2109630646-22371) [n:127.0.0.1:43420_t_zi    
 ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43420_t_zi&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1496557 INFO  
(OverseerThreadFactory-5832-thread-1-processing-n:127.0.0.1:43420_t_zi) 
[n:127.0.0.1:43420_t_zi     ] o.a.s.c.a.c.CreateCollectionCmd Create collection 
control_collection
   [junit4]   2> 1496665 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi    
x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core 
create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1496666 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi    
x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault 
Allocating transient cache for 4 transient cores
   [junit4]   2> 1497682 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.5.0
   [junit4]   2> 1497704 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1497831 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1497853 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' 
using configuration from configset conf1, trusted=true
   [junit4]   2> 1497853 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.core.control_collection.shard1.replica_n1' (registry 
'solr.core.control_collection.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1cc48781
   [junit4]   2> 1497866 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory 
solr.hdfs.home=hdfs://lucene2-us-west.apache.org:42008/solr_hdfs_home
   [junit4]   2> 1497866 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1497866 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore 
at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsSyncSliceTest_90B6D555A228D407-001/control-001/cores/control_collection_shard1_replica_n1],
 
dataDir=[hdfs://lucene2-us-west.apache.org:42008/solr_hdfs_home/control_collection/core_node2/data/]
   [junit4]   2> 1497868 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://lucene2-us-west.apache.org:42008/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
   [junit4]   2> 1497885 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 1497885 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[33554432] will allocate [1] slabs and use ~[33554432] bytes
   [junit4]   2> 1497885 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
   [junit4]   2> 1497984 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1 ] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1497991 INFO  (qtp2109630646-22370) [n:127.0.0.1:43420_t_zi 
c:control_collection s:shard1  x:control_collection_shard1_replica_n1

[...truncated too long message...]

6145807 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.w.WebAppContext@1b117c8c{datanode,/,null,UNAVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 6145807 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@306f7741{HTTP/1.1,[http/1.1]}{localhost:0}
   [junit4]   2> 6145807 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 6145807 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@fd14497{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
   [junit4]   2> 6145810 WARN  (BP-1284233050-127.0.0.1-1584479126712 
heartbeating to lucene2-us-west.apache.org/127.0.0.1:35043) [     ] 
o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager 
interrupted
   [junit4]   2> 6145811 WARN  (BP-1284233050-127.0.0.1-1584479126712 
heartbeating to lucene2-us-west.apache.org/127.0.0.1:35043) [     ] 
o.a.h.h.s.d.DataNode Ending block pool service for: Block pool 
BP-1284233050-127.0.0.1-1584479126712 (Datanode Uuid 
355f5d3e-9642-4ca6-8a09-a827dc6bf077) service to 
lucene2-us-west.apache.org/127.0.0.1:35043
   [junit4]   2> 6145835 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.w.WebAppContext@1e53208f{hdfs,/,null,UNAVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 6145835 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@2cf60e9f{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:0}
   [junit4]   2> 6145835 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 6145835 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[90B6D555A228D407]-worker) [     ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@b1970b{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.5/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_90B6D555A228D407-001
   [junit4]   2> Mar 17, 2020 9:13:52 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 32 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene84): 
{other_tl1=PostingsFormat(name=Asserting), 
range_facet_l_dv=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)),
 SubjectTerms_mfacet=PostingsFormat(name=Asserting), 
n_l1=PostingsFormat(name=LuceneVarGapFixedInterval), 
multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), 
intDefault=PostingsFormat(name=Asserting), 
n_td1=PostingsFormat(name=Asserting), 
n_d1=PostingsFormat(name=LuceneVarGapFixedInterval), 
range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), 
n_f1=PostingsFormat(name=Asserting), n_tl1=PostingsFormat(name=Asserting), 
oddField_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)),
 n_tf1=PostingsFormat(name=LuceneVarGapFixedInterval), 
a_t=PostingsFormat(name=LuceneVarGapFixedInterval), 
id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), 
text=Lucene84, timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), 
_root_=PostingsFormat(name=Asserting), n_dt1=PostingsFormat(name=Asserting), 
a_i1=PostingsFormat(name=LuceneVarGapFixedInterval), 
n_ti1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), 
rnd_b=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), 
foo_b=Lucene84, n_tdt1=PostingsFormat(name=Asserting), 
id_i1=PostingsFormat(name=Asserting), 
foo_d=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), 
range_facet_i_dv=PostingsFormat(name=LuceneVarGapFixedInterval), 
foo_f=Lucene84}, docValues:{other_tl1=DocValuesFormat(name=Lucene80), 
range_facet_l_dv=DocValuesFormat(name=Asserting), 
SubjectTerms_mfacet=DocValuesFormat(name=Lucene80), 
n_l1=DocValuesFormat(name=Direct), multiDefault=DocValuesFormat(name=Direct), 
intDefault=DocValuesFormat(name=Lucene80), 
n_td1=DocValuesFormat(name=Lucene80), n_d1=DocValuesFormat(name=Direct), 
range_facet_l=DocValuesFormat(name=Direct), 
n_f1=DocValuesFormat(name=Lucene80), n_tl1=DocValuesFormat(name=Lucene80), 
n_tf1=DocValuesFormat(name=Direct), a_t=DocValuesFormat(name=Direct), 
id=DocValuesFormat(name=Asserting), text=DocValuesFormat(name=Lucene80), 
intDvoDefault=DocValuesFormat(name=Asserting), 
timestamp=DocValuesFormat(name=Direct), _root_=DocValuesFormat(name=Lucene80), 
n_dt1=DocValuesFormat(name=Lucene80), a_i1=DocValuesFormat(name=Direct), 
n_ti1=DocValuesFormat(name=Asserting), rnd_b=DocValuesFormat(name=Asserting), 
_version_=DocValuesFormat(name=Lucene80), foo_b=DocValuesFormat(name=Lucene80), 
n_tdt1=DocValuesFormat(name=Lucene80), id_i1=DocValuesFormat(name=Lucene80), 
foo_d=DocValuesFormat(name=Asserting), 
range_facet_i_dv=DocValuesFormat(name=Direct), 
foo_f=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=349, 
maxMBSortInHeap=7.838900379157813, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3d29833e),
 locale=sv, timezone=America/Moncton
   [junit4]   2> NOTE: Linux 4.4.0-170-generic amd64/Oracle Corporation 
1.8.0_191 (64-bit)/cpus=8,threads=4,free=152621368,total=523239424
   [junit4]   2> NOTE: All tests run in this JVM: [AutoScalingHandlerTest, 
TestHashQParserPlugin, EmbeddedSolrNoSerializeTest, 
TestBackupRepositoryFactory, SpellingQueryConverterTest, SolrCLIZkUtilsTest, 
TestSolrJacksonAnnotation, DistributedQueueTest, TestFieldResource, 
TestCSVLoader, CollectionsAPISolrJTest, RestartWhileUpdatingTest, 
TestSortByMinMaxFunction, NodeLostTriggerIntegrationTest, 
TestPerFieldSimilarity, CdcrVersionReplicationTest, TestRestManager, 
CoreSorterTest, ByteBuffersDirectoryFactoryTest, 
TestTrackingShardHandlerFactory, TestInitParams, 
TlogReplayBufferedWhileIndexingTest, TestStressLiveNodes, SOLR749Test, 
TestJsonFacetsStatsParsing, TestFastLRUCache, RangeFacetCloudTest, 
TestExactSharedStatsCache, TestStressUserVersions, 
CollectionsAPIDistributedZkTest, CdcrOpsAndBoundariesTest, 
RegexBoostProcessorTest, SolrInfoBeanTest, AtomicUpdateRemovalJavabinTest, 
TestFiltering, TestSubQueryTransformerDistrib, TestCloudInspectUtil, 
TestCustomDocTransformer, TestXIncludeConfig, JWTAuthPluginTest, 
TestSha256AuthenticationProvider, TestConfig, TestWordDelimiterFilterFactory, 
TermVectorComponentTest, DistributedQueryElevationComponentTest, 
SolrMetricManagerTest, HdfsAutoAddReplicasIntegrationTest, TestJettySolrRunner, 
TestXmlQParser, TestRandomFaceting, TestUtils, TestCoreAdminApis, 
MissingSegmentRecoveryTest, TaggerTest, GraphQueryTest, 
TestLMJelinekMercerSimilarityFactory, TestNumericRangeQuery64, 
BasicFunctionalityTest, TestMergePolicyConfig, ConnectionManagerTest, 
HttpPartitionWithTlogReplicasTest, TestDeprecatedFilters, TestApiFramework, 
TestInPlaceUpdatesDistrib, SmileWriterTest, TestTermsQParserPlugin, 
TestSolrCoreSnapshots, TestJsonRequest, TestSQLHandler, 
ChaosMonkeySafeLeaderWithPullReplicasTest, HdfsThreadLeakTest, 
TestReplicationHandlerDiskOverFlow, DistributedMLTComponentTest, 
ShardsWhitelistTest, DistanceUnitsTest, 
IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestMultiWordSynonyms, 
TestSimNodeLostTrigger, ReplaceNodeNoTargetTest, SSLMigrationTest, 
TestDynamicFieldCollectionResource, TestPrepRecovery, TestSchemaManager, 
DistribCursorPagingTest, DistributedFacetPivotSmallAdvancedTest, 
TestPartialUpdateDeduplication, ChangedSchemaMergeTest, SchemaApiFailureTest, 
TestDistributedSearch, DocValuesMultiTest, HealthCheckHandlerTest, 
ZkFailoverTest, SolrSlf4jReporterTest, TestMaxScoreQueryParser, 
DebugComponentTest, TestConfigSetProperties, 
PeerSyncWithLeaderAndIndexFingerprintCachingTest, CurrencyRangeFacetCloudTest, 
RequestLoggingTest, TestSolrTestCaseJ4, TestSolrQueryResponse, 
TestSolrDeletionPolicy2, ResponseHeaderTest, ImplicitSnitchTest, 
TestCollationField, TestJavabinTupleStreamParser, TestDirectoryFactory, 
BlockCacheTest, TestConfigSetsAPIExclusivity, MultiDestinationAuditLoggerTest, 
TestSegmentSorting, DistributedUpdateProcessorTest, BadCopyFieldTest, 
TestDownShardTolerantSearch, IndexSizeEstimatorTest, SpatialRPTFieldTypeTest, 
TestRebalanceLeaders, TestStressLucene, TestSimClusterStateProvider, 
JsonLoaderTest, MetricsHistoryIntegrationTest, BasicAuthOnSingleNodeTest, 
TestHdfsCloudBackupRestore, IndexSizeTriggerSizeEstimationTest, 
MetricTriggerTest, V2ApiIntegrationTest, TestDistributedTracing, 
TestLegacyField, AnalysisErrorHandlingTest, TestFieldCacheSort, 
TestFieldCacheWithThreads, FullHLLTest, UninvertDocValuesMergePolicyTest, 
TestUnifiedSolrHighlighter, OverriddenZkACLAndCredentialsProvidersTest, 
TestDeleteCollectionOnDownNodes, PackageManagerCLITest, TestSolr4Spatial, 
TestSimTriggerIntegration, FastVectorHighlighterTest, MoveReplicaTest, 
ManagedSchemaRoundRobinCloudTest, TestComplexPhraseQParserPlugin, 
TestStreamBody, RAMDirectoryFactoryTest, HdfsWriteToMultipleCollectionsTest, 
TestHighlightDedupGrouping, TestPayloadCheckQParserPlugin, TestPolicyCloud, 
TriggerEventQueueTest, TriggerIntegrationTest, TestSnapshotCloudManager, 
BaseCdcrDistributedZkTest, CdcrBidirectionalTest, HDFSCollectionsAPITest, 
HdfsBasicDistributedZk2Test]
   [junit4] Completed [900/900 (3!)] on J1 in 508.70s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 52546 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to