Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/14469/
Java: 32bit/jdk1.8.0_60 -server -XX:+UseG1GC

2 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudCluster

Error Message:
21 threads leaked from SUITE scope at 
org.apache.solr.cloud.TestMiniSolrCloudCluster:     1) Thread[id=886, 
name=qtp30551041-886, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] 
        at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=988, 
name=coreContainerWorkExecutor-232-thread-1, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=887, 
name=Scheduler-15357840, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=957, 
name=jetty-launcher-128-thread-3-EventThread, state=WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    
5) Thread[id=984, name=Thread-387, state=WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at java.lang.Object.wait(Native 
Method)         at java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)    6) 
Thread[id=963, name=zkCallback-143-thread-1, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=884, 
name=qtp30551041-884, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] 
        at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)    8) Thread[id=888, 
name=org.eclipse.jetty.server.session.HashSessionManager@73b3fTimer, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=1008, 
name=zkCallback-143-thread-3, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=881, 
name=qtp30551041-881-selector-ServerConnectorManager@b752/2, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   11) Thread[id=1006, 
name=OverseerCollectionConfigSetProcessor-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)        
 at 
org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:355)
         at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:172) 
        at java.lang.Thread.run(Thread.java:745)   12) Thread[id=1009, 
name=zkCallback-143-thread-4, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   13) Thread[id=883, 
name=qtp30551041-883-acceptor-0@136519b-ServerConnector@16542f{HTTP/1.1}{127.0.0.1:44210},
 state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   14) Thread[id=1005, 
name=OverseerStateUpdate-94661094469599243-127.0.0.1:44210_solr-n_0000000004, 
state=TIMED_WAITING, group=Overseer state updater.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:333)    
     at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:310)
         at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:289)    
     at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:358)  
       at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:143)       
  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)  
       at java.lang.Thread.run(Thread.java:745)   15) Thread[id=1007, 
name=OverseerHdfsCoreFailoverThread-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)         
at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:725)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=1002, 
name=zkCallback-143-thread-2, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   17) Thread[id=880, 
name=qtp30551041-880-selector-ServerConnectorManager@b752/1, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   18) Thread[id=879, 
name=qtp30551041-879-selector-ServerConnectorManager@b752/0, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   19) Thread[id=882, 
name=qtp30551041-882-selector-ServerConnectorManager@b752/3, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   20) Thread[id=885, 
name=qtp30551041-885, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] 
        at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   21) Thread[id=956, 
name=jetty-launcher-128-thread-3-SendThread(127.0.0.1:51578), 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 21 threads leaked from 
SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudCluster: 
   1) Thread[id=886, name=qtp30551041-886, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=988, name=coreContainerWorkExecutor-232-thread-1, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=887, name=Scheduler-15357840, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=957, name=jetty-launcher-128-thread-3-EventThread, 
state=WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   5) Thread[id=984, name=Thread-387, state=WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)
   6) Thread[id=963, name=zkCallback-143-thread-1, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=884, name=qtp30551041-884, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=888, 
name=org.eclipse.jetty.server.session.HashSessionManager@73b3fTimer, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=1008, name=zkCallback-143-thread-3, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=881, 
name=qtp30551041-881-selector-ServerConnectorManager@b752/2, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=1006, 
name=OverseerCollectionConfigSetProcessor-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
        at 
org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:355)
        at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:172)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=1009, name=zkCallback-143-thread-4, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=883, 
name=qtp30551041-883-acceptor-0@136519b-ServerConnector@16542f{HTTP/1.1}{127.0.0.1:44210},
 state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)
        at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=1005, 
name=OverseerStateUpdate-94661094469599243-127.0.0.1:44210_solr-n_0000000004, 
state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:333)
        at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:310)
        at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:289)
        at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:358)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:143)
        at 
org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=1007, 
name=OverseerHdfsCoreFailoverThread-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
        at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:725)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=1002, name=zkCallback-143-thread-2, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=880, 
name=qtp30551041-880-selector-ServerConnectorManager@b752/1, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=879, 
name=qtp30551041-879-selector-ServerConnectorManager@b752/0, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=882, 
name=qtp30551041-882-selector-ServerConnectorManager@b752/3, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  20) Thread[id=885, name=qtp30551041-885, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)
        at java.lang.Thread.run(Thread.java:745)
  21) Thread[id=956, 
name=jetty-launcher-128-thread-3-SendThread(127.0.0.1:51578), 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
        at __randomizedtesting.SeedInfo.seed([4DD94E84D84F5AD3]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudCluster

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=988, name=coreContainerWorkExecutor-232-thread-1, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=887, 
name=Scheduler-15357840, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=984, 
name=Thread-387, state=WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)    4) 
Thread[id=963, name=zkCallback-143-thread-1, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=888, 
name=org.eclipse.jetty.server.session.HashSessionManager@73b3fTimer, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=1008, 
name=zkCallback-143-thread-3, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=881, 
name=qtp30551041-881-selector-ServerConnectorManager@b752/2, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.nio.ch.IOUtil.drain(Native 
Method)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)    8) Thread[id=1009, 
name=zkCallback-143-thread-4, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=1007, 
name=OverseerHdfsCoreFailoverThread-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=1002, 
name=zkCallback-143-thread-2, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   11) Thread[id=880, 
name=qtp30551041-880-selector-ServerConnectorManager@b752/1, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at sun.nio.ch.IOUtil.drain(Native 
Method)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   12) Thread[id=879, 
name=qtp30551041-879-selector-ServerConnectorManager@b752/0, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   13) Thread[id=882, 
name=qtp30551041-882-selector-ServerConnectorManager@b752/3, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   14) Thread[id=956, 
name=jetty-launcher-128-thread-3-SendThread(127.0.0.1:51578), 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=988, name=coreContainerWorkExecutor-232-thread-1, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=887, name=Scheduler-15357840, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=984, name=Thread-387, state=WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)
   4) Thread[id=963, name=zkCallback-143-thread-1, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=888, 
name=org.eclipse.jetty.server.session.HashSessionManager@73b3fTimer, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=1008, name=zkCallback-143-thread-3, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=881, 
name=qtp30551041-881-selector-ServerConnectorManager@b752/2, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=1009, name=zkCallback-143-thread-4, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=1007, 
name=OverseerHdfsCoreFailoverThread-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=1002, name=zkCallback-143-thread-2, state=TIMED_WAITING, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=880, 
name=qtp30551041-880-selector-ServerConnectorManager@b752/1, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=879, 
name=qtp30551041-879-selector-ServerConnectorManager@b752/0, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=882, 
name=qtp30551041-882-selector-ServerConnectorManager@b752/3, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
        at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
        at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=956, 
name=jetty-launcher-128-thread-3-SendThread(127.0.0.1:51578), 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
        at __randomizedtesting.SeedInfo.seed([4DD94E84D84F5AD3]:0)




Build Log:
[...truncated 9235 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestMiniSolrCloudCluster
   [junit4]   2> 0    INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 7    INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 7    INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 157  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.ZkTestServer start zk server on port:58272
   [junit4]   2> 175  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 237  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 349  INFO  (zkCallback-2-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@7d5873 name:ZooKeeperConnection 
Watcher:127.0.0.1:58272 got event WatchedEvent state:SyncConnected type:None 
path:null path:null type:None
   [junit4]   2> 350  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 352  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 367  INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 463  INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.u.log 
Logging initialized @2118ms
   [junit4]   2> 665  INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 666  INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 829  INFO  (jetty-launcher-1-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1ac0b2{/solr,null,AVAILABLE}
   [junit4]   2> 829  INFO  (jetty-launcher-1-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1564fa3{/solr,null,AVAILABLE}
   [junit4]   2> 837  INFO  (jetty-launcher-1-thread-3) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@2cde49{HTTP/1.1}{127.0.0.1:53419}
   [junit4]   2> 837  INFO  (jetty-launcher-1-thread-3) [    ] o.e.j.s.Server 
Started @2497ms
   [junit4]   2> 838  INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=53419}
   [junit4]   2> 840  INFO  (jetty-launcher-1-thread-1) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@c3bd20{HTTP/1.1}{127.0.0.1:35326}
   [junit4]   2> 841  INFO  (jetty-launcher-1-thread-1) [    ] o.e.j.s.Server 
Started @2500ms
   [junit4]   2> 841  INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=35326}
   [junit4]   2> 843  INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): 
sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 843  INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): 
sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 868  INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: 
'/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/'
   [junit4]   2> 868  INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: 
'/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/'
   [junit4]   2> 872  INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 872  INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 872  INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find 
system property or JNDI)
   [junit4]   2> 872  INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find 
system property or JNDI)
   [junit4]   2> 1013 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1013 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1015 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1016 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1024 INFO  (zkCallback-4-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@1208088 name:ZooKeeperConnection 
Watcher:127.0.0.1:58272/solr got event WatchedEvent state:SyncConnected 
type:None path:null path:null type:None
   [junit4]   2> 1025 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1026 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1026 INFO  (zkCallback-3-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@18da140 name:ZooKeeperConnection 
Watcher:127.0.0.1:58272/solr got event WatchedEvent state:SyncConnected 
type:None path:null path:null type:None
   [junit4]   2> 1026 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1027 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1027 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1027 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1237 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.CoresLocator Config-defined core root directory: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/.
   [junit4]   2> 1245 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.CoresLocator Config-defined core root directory: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/.
   [junit4]   2> 1290 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.CoreContainer New CoreContainer 12443085
   [junit4]   2> 1290 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.CoreContainer New CoreContainer 4766803
   [junit4]   2> 1291 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.CoreContainer Loading cores into CoreContainer 
[instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/]
   [junit4]   2> 1291 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.CoreContainer loading shared library: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/lib
   [junit4]   2> 1292 WARN  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to 
classloader: lib (resolved as: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/lib).
   [junit4]   2> 1305 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.CoreContainer Loading cores into CoreContainer 
[instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/]
   [junit4]   2> 1305 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.CoreContainer loading shared library: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/lib
   [junit4]   2> 1305 WARN  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to 
classloader: lib (resolved as: 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/lib).
   [junit4]   2> 1346 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme 
: ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 
10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 
5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1347 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme 
: ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 
10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 
5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1588 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1588 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1591 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1591 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1593 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.l.LogWatcher Registering Log Listener [Log4j 
(org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1593 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.l.LogWatcher Registering Log Listener [Log4j 
(org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1594 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58272/solr
   [junit4]   2> 1594 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58272/solr
   [junit4]   2> 1607 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 1607 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 1607 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1607 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1608 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1608 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1615 INFO  (zkCallback-7-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@bbdc7b name:ZooKeeperConnection 
Watcher:127.0.0.1:58272 got event WatchedEvent state:SyncConnected type:None 
path:null path:null type:None
   [junit4]   2> 1615 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1616 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1617 INFO  (zkCallback-8-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@24827e name:ZooKeeperConnection 
Watcher:127.0.0.1:58272 got event WatchedEvent state:SyncConnected type:None 
path:null path:null type:None
   [junit4]   2> 1617 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1618 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1725 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1727 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1727 INFO  
(zkCallback-9-thread-1-processing-n:127.0.0.1:53419_solr) 
[n:127.0.0.1:53419_solr    ] o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@19159c8 name:ZooKeeperConnection 
Watcher:127.0.0.1:58272/solr got event WatchedEvent state:SyncConnected 
type:None path:null path:null type:None
   [junit4]   2> 1728 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1729 INFO  
(zkCallback-10-thread-1-processing-n:127.0.0.1:35326_solr) 
[n:127.0.0.1:35326_solr    ] o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@12ade06 name:ZooKeeperConnection 
Watcher:127.0.0.1:58272/solr got event WatchedEvent state:SyncConnected 
type:None path:null path:null type:None
   [junit4]   2> 1729 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1762 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 1763 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 1774 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 1775 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 1799 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 1799 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 1814 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 1853 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 1853 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 1870 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1878 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1882 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1882 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1889 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1890 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1895 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1896 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1907 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1918 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:53419_solr
   [junit4]   2> 1922 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:53419_solr
   [junit4]   2> 1929 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35326_solr
   [junit4]   2> 1930 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 1932 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:35326_solr
   [junit4]   2> 1942 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 1943 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 1946 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1946 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1949 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.LeaderElector Joined leadership election with path: 
/overseer_elect/election/94661087357501446-127.0.0.1:35326_solr-n_0000000000
   [junit4]   2> 1949 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.LeaderElector Joined leadership election with path: 
/overseer_elect/election/94661087357501445-127.0.0.1:53419_solr-n_0000000001
   [junit4]   2> 1955 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.OverseerElectionContext I am going to be the leader 
127.0.0.1:35326_solr
   [junit4]   2> 1961 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.LeaderElector Watching path 
/overseer_elect/election/94661087357501446-127.0.0.1:35326_solr-n_0000000000 to 
know if I could be the leader
   [junit4]   2> 1961 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1961 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 1969 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.Overseer Overseer 
(id=94661087357501446-127.0.0.1:35326_solr-n_0000000000) starting
   [junit4]   2> 1980 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 2070 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting 
OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 
autoReplicaFailoverWaitAfterExpiration=10000 
autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 2076 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for 
authorization module.
   [junit4]   2> 2076 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 2143 INFO  
(OverseerCollectionConfigSetProcessor-94661087357501446-127.0.0.1:35326_solr-n_0000000000)
 [n:127.0.0.1:35326_solr    ] o.a.s.c.OverseerTaskProcessor Process current 
queue of overseer operations
   [junit4]   2> 2143 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 2147 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.CoresLocator Looking for core definitions underneath 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/.
   [junit4]   2> 2149 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.c.CoresLocator Found 0 core definitions
   [junit4]   2> 2151 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.s.SolrDispatchFilter 
user.dir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2
   [junit4]   2> 2152 INFO  (jetty-launcher-1-thread-3) [n:127.0.0.1:53419_solr 
   ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 2158 INFO  
(OverseerStateUpdate-94661087357501446-127.0.0.1:35326_solr-n_0000000000) 
[n:127.0.0.1:35326_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 2212 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for 
authorization module.
   [junit4]   2> 2212 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 2213 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.CoresLocator Looking for core definitions underneath 
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-001/.
   [junit4]   2> 2214 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.c.CoresLocator Found 0 core definitions
   [junit4]   2> 2229 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.s.SolrDispatchFilter 
user.dir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2
   [junit4]   2> 2229 INFO  (jetty-launcher-1-thread-1) [n:127.0.0.1:35326_solr 
   ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 2238 INFO  (jetty-launcher-1-thread-1) [    ] 
o.e.j.s.ServerConnector Stopped ServerConnector@2cde49{HTTP/1.1}{127.0.0.1:0}
   [junit4]   2> 2238 INFO  (jetty-launcher-1-thread-1) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=4766803
   [junit4]   2> 2241 INFO  (jetty-launcher-1-thread-3) [    ] 
o.e.j.s.ServerConnector Stopped ServerConnector@c3bd20{HTTP/1.1}{127.0.0.1:0}
   [junit4]   2> 2241 INFO  (jetty-launcher-1-thread-3) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=12443085
   [junit4]   2> 2243 INFO  (jetty-launcher-1-thread-3) [    ] o.a.s.c.Overseer 
Overseer (id=94661087357501446-127.0.0.1:35326_solr-n_0000000000) closing
   [junit4]   2> 2245 INFO  
(OverseerStateUpdate-94661087357501446-127.0.0.1:35326_solr-n_0000000000) 
[n:127.0.0.1:35326_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:35326_solr
   [junit4]   2> 2253 INFO  
(zkCallback-10-thread-1-processing-n:127.0.0.1:35326_solr) 
[n:127.0.0.1:35326_solr    ] o.a.s.c.c.ZkStateReader A live node change: 
WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has 
occurred - updating... (live nodes size: 2)
   [junit4]   2> 2254 WARN  
(zkCallback-10-thread-1-processing-n:127.0.0.1:35326_solr) 
[n:127.0.0.1:35326_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK
   [junit4]   2> 2256 ERROR (jetty-launcher-1-thread-3-EventThread) 
[n:127.0.0.1:53419_solr    ] o.a.z.ClientCnxn Error while calling watcher 
   [junit4]   2> java.util.concurrent.RejectedExecutionException: Task 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1@159455c 
rejected from 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@166d06f[Terminated,
 pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
   [junit4]   2>        at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2>        at 
org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:266)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 2257 INFO  (jetty-launcher-1-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1ac0b2{/solr,null,UNAVAILABLE}
   [junit4]   2> 2256 INFO  (jetty-launcher-1-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1564fa3{/solr,null,UNAVAILABLE}
   [junit4]   2> 2265 INFO  
(TEST-TestMiniSolrCloudCluster.testErrorsInStartup-seed#[4DD94E84D84F5AD3]) [   
 ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:58272 58272
   [junit4]   2> 2290 INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer connecting 
to 127.0.0.1:58272 58272
   [junit4]   2> 2305 WARN  (Thread-1) [    ] o.a.s.c.ZkTestServer Watch limit 
violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        2       /solr/aliases.json
   [junit4]   2>        2       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        2       /solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        2       /solr/live_nodes
   [junit4]   2>        2       /solr/collections
   [junit4]   2> 
   [junit4]   2> 2347 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2354 INFO  (Thread-25) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2354 INFO  (Thread-25) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 2454 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.ZkTestServer start zk server on port:39942
   [junit4]   2> 2454 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 2454 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2480 INFO  (zkCallback-12-thread-1) [    ] 
o.a.s.c.c.ConnectionManager Watcher 
org.apache.solr.common.cloud.ConnectionManager@12d7f82 name:ZooKeeperConnection 
Watcher:127.0.0.1:39942 got event WatchedEvent state:SyncConnected type:None 
path:null path:null type:None
   [junit4]   2> 2480 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2481 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 2481 INFO  
(TEST-TestMiniSolrCloudCluster.testCollectionCreateWithoutCoresThenDelete-seed#[4DD94E84D84F5AD3])
 [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 2502 INFO  (jetty-launcher-11-thread-1) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 2505 INFO  (jetty-launcher-11-thread-3) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 2506 INFO  (jetty-launcher-11-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@12d3821{/solr,null,AVAILABLE}
   [junit4]   2> 2507 INFO  (jetty-launcher-11-thread-1) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@e21981{HTTP/1.1}{127.0.0.1:33842}
   [junit4]   2> 2507 INFO  (jetty-launcher-11-thread-1) [    ] o.e.j.s.Server 
Started @4167ms
   [junit4]   2> 2507 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=33842}
   [junit4]   2> 2508 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): 
sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 2509 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: 
'/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-002/'
   [junit4]   2> 2509 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2509 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find 
system property or JNDI)
   [junit4]   2> 2514 INFO  (jetty-launcher-11-thread-4) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 2515 INFO  (jetty-launcher-11-thread-4) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@148018b{/solr,null,AVAILABLE}
   [junit4]   2> 2529 INFO  (jetty-launcher-11-thread-4) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@1325705{HTTP/1.1}{127.0.0.1:60790}
   [junit4]   2> 2529 INFO  (jetty-launcher-11-thread-4) [    ] o.e.j.s.Server 
Started @4189ms
   [junit4]   2> 2529 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=60790}
   [junit4]   2> 2530 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): 
sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 2530 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: 
'/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestMiniSolrCloudCluster_4DD94E84D84F5AD3-001/tempDir-002/'
   [junit4]   2> 2530 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2530 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find 
system property or JNDI)
   [junit4]   2> 2537 INFO  (jetty-launcher-11-thread-5) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 2571 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 2578 INFO  (jetty-launcher-11-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1499a6c{/solr,null,AVAILABLE}
   [junit4]   2> 2579 INFO  (jetty-launcher-11-thread-3) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@e057ca{HTTP/1.1}{127.0.0.1:33491}
   [junit4]   2> 2586 INFO  (jetty-launcher-11-thread-2) [    ] o.e.j.s.Server 
jetty-9.2.13.v20150730
   [junit4]   2> 2587 INFO  (jetty-launcher-11-thread-5) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1075aba{/solr,null,AVAILABLE}
   [junit4]   2> 2589 INFO  (jetty-launcher-11-thread-3) [    ] o.e.j.s.Server 
Started @4247ms
   [junit4]   2> 2589 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=33491}
   [junit4]   2> 2590 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): 
sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 2590 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: 
'/home/jenkins/workspace/Lucene-Solr-trunk

[...truncated too long message...]

nousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=881, 
name=qtp30551041-881-selector-ServerConnectorManager@b752/2, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    8) Thread[id=1009, name=zkCallback-143-thread-4, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    9) Thread[id=1007, 
name=OverseerHdfsCoreFailoverThread-94661094469599243-127.0.0.1:44210_solr-n_0000000004,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=1002, name=zkCallback-143-thread-2, 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   11) Thread[id=880, 
name=qtp30551041-880-selector-ServerConnectorManager@b752/1, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=879, 
name=qtp30551041-879-selector-ServerConnectorManager@b752/0, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native 
Method)
   [junit4]    >         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   13) Thread[id=882, 
name=qtp30551041-882-selector-ServerConnectorManager@b752/3, state=RUNNABLE, 
group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at sun.nio.ch.EPollArrayWrapper.epollWait(Native 
Method)
   [junit4]    >         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
   [junit4]    >         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   14) Thread[id=956, 
name=jetty-launcher-128-thread-3-SendThread(127.0.0.1:51578), 
state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([4DD94E84D84F5AD3]:0)
   [junit4] Completed [33/547] on J2 in 141.13s, 5 tests, 2 errors <<< FAILURES!

[...truncated 1587 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:775: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:719: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:59: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:233: The 
following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:516: The 
following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1432: 
The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:987: 
There were test failures: 547 suites (8 ignored), 2148 tests, 2 suite-level 
errors, 1393 ignored (23 assumptions) [seed: 4DD94E84D84F5AD3]

Total time: 44 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to