[JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 715 - Still Failing

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/715/

4 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
323 threads leaked from SUITE scope at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=17732, 
name=searcherExecutor-4411-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=18458, 
name=qtp699233344-18458, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=16148, 
name=ProcessThread(sid:0 cport:-1):, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
    4) Thread[id=18243, name=coreZkRegister-3175-thread-18, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=18400, 
name=coreZkRegister-3349-thread-40-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:955)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=15977, 
name=searcherExecutor-3253-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=18300, 
name=coreZkRegister-3197-thread-29, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    8) Thread[id=18188, 
name=coreZkRegister-3186-thread-9-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=18359, 
name=coreZkRegister-3186-thread-25, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=18343, 
name=zkCallback-909-thread-29, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   11) Thread[id=19433, 
name=searcherExecutor-4568-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   12) Thread[id=19223, 
name=coreZkRegister-3197-thread-64, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   13) Thread[id=17638, 
name=searcherExecutor-4384-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   14) Thread[id=17680, 
name=searcherExecutor-4395-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   15) Thread[id=19356, 
name=coreZkRegister-3349-thread-68-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard5_replica1 s:shard5 
c:awholynewstresscollection_collection2_4 r:core_node32, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:460)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:224)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   16) Thread[id=18186, 
name=coreZkRegister-3186-thread-7-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   17) Thread[id=16123, 
name=searcherExecutor-3366-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   18) Thread[id=17514, 
name=qtp215754624-17514, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   19) Thread[id=19729, 
name=RecoveryThread-awholynewstresscollection_collection5_4_shard7_replica4, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)         at 
java.util.concurrent.FutureTask.get(FutureTask.java:191)         at 
org.apache.solr.cloud.RecoveryStrategy.sendPrepRecoveryCmd(RecoveryStrategy.java:598)
         at 
org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:361)    
     at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:227)   
20) Thread[id=18661, name=qtp639560314-18661, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   21) Thread[id=15920, 
name=searcherExecutor-3220-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   22) Thread[id=17454, 
name=qtp1764830233-17454, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   23) Thread[id=19277, 
name=coreZkRegister-3349-thread-57-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard5_replica1 s:shard5 
c:awholynewstresscollection_collection2_4 r:core_node32, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:460)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:224)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   24) Thread[id=15963, 
name=searcherExecutor-3240-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   25) Thread[id=19547, 
name=searcherExecutor-4620-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   26) Thread[id=19688, 
name=qtp639560314-19688, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   27) Thread[id=15862, 
name=org.eclipse.jetty.server.session.HashSessionManager@49ef4326Timer, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   28) Thread[id=19352, 
name=coreZkRegister-3349-thread-64, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   29) Thread[id=18132, 
name=coreZkRegister-3349-thread-24, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   30) Thread[id=18555, 
name=qtp699233344-18555, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   31) Thread[id=19125, 
name=zkCallback-873-thread-15-processing-n:127.0.0.1:48979__wgo-EventThread, 
state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
32) Thread[id=19696, 
name=zkCallback-873-thread-76-processing-n:127.0.0.1:48979__wgo, 
state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   33) Thread[id=18128, 
name=coreZkRegister-3349-thread-20-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   34) Thread[id=18275, 
name=zkCallback-909-thread-25-processing-n:127.0.0.1:44719__wgo, state=BLOCKED, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   35) Thread[id=15861, 
name=Scheduler-393793085, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   36) Thread[id=18278, 
name=zkCallback-876-thread-32, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   37) Thread[id=17470, 
name=qtp639560314-17470, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:460)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:224)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at org.apache.solr.cloud.ZkController.register(ZkController.java:839)         
at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:187)         at 
org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:216)         at 
org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:629)         
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:683)         at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:649)         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:628)
         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:213)
         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:193)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
         at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:654)  
       at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:433)      
   at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:227)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:196)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:106)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)    
     at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)     
    at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)  
       at org.eclipse.jetty.server.Server.handle(Server.java:499)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)     
    at 
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)      
   at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   38) Thread[id=19119, 
name=zkCallback-873-thread-15-processing-n:127.0.0.1:48979__wgo-SendThread(127.0.0.1:44215),
 state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   39) 
Thread[id=19210, 
name=coreZkRegister-3197-thread-59-processing-n:127.0.0.1:33469__wgo 
x:awholynewstresscollection_collection2_4_shard7_replica2 s:shard7 
c:awholynewstresscollection_collection2_4 r:core_node12, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   40) Thread[id=18232, 
name=coreZkRegister-3175-thread-9, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   41) Thread[id=17551, 
name=qtp639560314-17551, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   42) Thread[id=19173, 
name=coreZkRegister-3186-thread-51, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   43) Thread[id=19244, 
name=coreZkRegister-3186-thread-52-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:460)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:224)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   44) Thread[id=19535, 
name=searcherExecutor-4605-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   45) Thread[id=16089, 
name=org.eclipse.jetty.server.session.HashSessionManager@48dd0c49Timer, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   46) Thread[id=19576, 
name=searcherExecutor-4640-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   47) Thread[id=19692, 
name=zkCallback-873-thread-73, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   48) Thread[id=19477, 
name=RecoveryThread-awholynewcollection_0_shard2_replica1, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:511)    
     at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:227)   
49) Thread[id=19172, name=coreZkRegister-3186-thread-50, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   50) Thread[id=16267, 
name=searcherExecutor-3414-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   51) Thread[id=17110, 
name=qtp639560314-17110, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   52) Thread[id=18389, 
name=coreZkRegister-3349-thread-32-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:683)
         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:652) 
        at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:980)  
       at org.apache.solr.cloud.ZkController.register(ZkController.java:896)    
     at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   53) Thread[id=19166, 
name=coreZkRegister-3186-thread-47-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=BLOCKED, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
org.apache.solr.update.DefaultSolrCoreState.doRecovery(DefaultSolrCoreState.java:299)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:478)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:330)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   54) Thread[id=19691, 
name=zkCallback-870-thread-64, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
org.apache.solr.common.cloud.ZkStateReader$7.process(ZkStateReader.java:938)    
     at 
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:269)        
 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)     
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   55) Thread[id=19391, 
name=searcherExecutor-4532-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   56) Thread[id=18387, 
name=coreZkRegister-3349-thread-30-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=BLOCKED, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
org.apache.solr.update.DefaultSolrCoreState.doRecovery(DefaultSolrCoreState.java:299)
         at 
org.apache.solr.cloud.ZkController.checkRecovery(ZkController.java:1095)        
 at org.apache.solr.cloud.ZkController.register(ZkController.java:929)         
at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   57) Thread[id=18199, 
name=coreZkRegister-3186-thread-20, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   58) Thread[id=15919, 
name=searcherExecutor-3219-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   59) Thread[id=15983, 
name=searcherExecutor-3282-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   60) Thread[id=18167, 
name=coreZkRegister-3197-thread-23, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   61) Thread[id=17206, 
name=searcherExecutor-4167-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   62) Thread[id=18361, 
name=coreZkRegister-3186-thread-27-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ActionThrottle.minimumWaitBetweenActions(ActionThrottle.java:82)
         at 
org.apache.solr.update.DefaultSolrCoreState.doRecovery(DefaultSolrCoreState.java:326)
         at 
org.apache.solr.cloud.ZkController.checkRecovery(ZkController.java:1095)        
 at org.apache.solr.cloud.ZkController.register(ZkController.java:929)         
at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   63) Thread[id=17499, 
name=qtp215754624-17499, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:955)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at org.apache.solr.cloud.ZkController.register(ZkController.java:839)        
 at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:187)         at 
org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:216)         at 
org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:629)         
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:683)         at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:649)         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:628)
         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:213)
         at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:193)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
         at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:654)  
       at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:433)      
   at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:227)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:196)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:106)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)    
     at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)     
    at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)  
       at org.eclipse.jetty.server.Server.handle(Server.java:499)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)     
    at 
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)      
   at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   64) Thread[id=19736, 
name=zkCallback-870-thread-67, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
org.apache.solr.common.cloud.ZkStateReader$7.process(ZkStateReader.java:938)    
     at 
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:269)        
 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)     
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   65) Thread[id=18116, 
name=coreZkRegister-3349-thread-11-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:955)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   66) Thread[id=18239, 
name=coreZkRegister-3175-thread-15-processing-n:127.0.0.1:43783__wgo 
x:awholynewstresscollection_collection2_4_shard2_replica1 s:shard2 
c:awholynewstresscollection_collection2_4 r:core_node9, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:683)
         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:652) 
        at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:980)  
       at org.apache.solr.cloud.ZkController.register(ZkController.java:896)    
     at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   67) Thread[id=19733, 
name=RecoveryThread-awholynewstresscollection_collection0_4_shard1_replica3, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)         at 
java.util.concurrent.FutureTask.get(FutureTask.java:191)         at 
org.apache.solr.cloud.RecoveryStrategy.sendPrepRecoveryCmd(RecoveryStrategy.java:598)
         at 
org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:361)    
     at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:227)   
68) Thread[id=19567, name=searcherExecutor-4663-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   69) Thread[id=19243, 
name=zkCallback-876-thread-77, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   70) Thread[id=18334, 
name=coreZkRegister-3197-thread-39-processing-n:127.0.0.1:33469__wgo 
x:awholynewstresscollection_collection2_4_shard7_replica2 s:shard7 
c:awholynewstresscollection_collection2_4 r:core_node12, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   71) Thread[id=16081, 
name=qtp639560314-16081-selector-ServerConnectorManager@759ecdc5/1, 
state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   72) Thread[id=19164, 
name=coreZkRegister-3186-thread-45-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)       
  at 
org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:136)
         at 
org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:315)
         at 
org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:199) 
        at 
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)    
     at 
org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)        
 at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)     
    at org.apache.solr.cloud.ZkController.register(ZkController.java:884)       
  at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   73) Thread[id=19293, 
name=coreZkRegister-3175-thread-26, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   74) Thread[id=19205, 
name=coreZkRegister-3197-thread-54, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   75) Thread[id=19162, 
name=coreZkRegister-3186-thread-43, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   76) Thread[id=18184, 
name=coreZkRegister-3186-thread-6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   77) Thread[id=17469, 
name=qtp215754624-17469, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   78) Thread[id=17490, 
name=qtp699233344-17490, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   79) Thread[id=19208, 
name=coreZkRegister-3197-thread-57, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   80) Thread[id=18229, 
name=coreZkRegister-3175-thread-6-processing-n:127.0.0.1:43783__wgo 
x:awholynewstresscollection_collection2_4_shard2_replica1 s:shard2 
c:awholynewstresscollection_collection2_4 r:core_node9, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:683)
         at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:652) 
        at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:959)  
       at org.apache.solr.cloud.ZkController.register(ZkController.java:896)    
     at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   81) Thread[id=19633, 
name=OverseerCollectionProcessor-94018458300252292-127.0.0.1:33469__wgo-n_0000000016,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
java.lang.Object.wait(Native Method)         at 
org.apache.solr.cloud.DistributedQueue$LatchWatcher.await(DistributedQueue.java:276)
         at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:320)   
      at 
org.apache.solr.cloud.DistributedQueue.peekTopN(DistributedQueue.java:447)      
   at 
org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:317)
         at java.lang.Thread.run(Thread.java:745)   82) Thread[id=18526, 
name=qtp699233344-18526, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   83) Thread[id=19735, 
name=zkCallback-870-thread-66, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
org.apache.solr.common.cloud.ZkStateReader$7.process(ZkStateReader.java:938)    
     at 
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:269)        
 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)     
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   84) Thread[id=18119, 
name=coreZkRegister-3349-thread-14-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   85) Thread[id=19171, 
name=coreZkRegister-3186-thread-49, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   86) Thread[id=18390, 
name=coreZkRegister-3349-thread-33-processing-n:127.0.0.1:44719__wgo 
x:awholynewstresscollection_collection2_4_shard8_replica4 s:shard8 
c:awholynewstresscollection_collection2_4 r:core_node13, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:955)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   87) Thread[id=19214, 
name=coreZkRegister-3197-thread-63, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   88) Thread[id=16104, 
name=Thread-11296, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]   
      at java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1095)   89) 
Thread[id=19247, 
name=coreZkRegister-3186-thread-54-processing-n:127.0.0.1:48979__wgo 
x:awholynewstresscollection_collection2_4_shard3_replica1 s:shard3 
c:awholynewstresscollection_collection2_4 r:core_node6, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ActionThrottle.minimumWaitBetweenActions(ActionThrottle.java:82)
         at 
org.apache.solr.update.DefaultSolrCoreState.doRecovery(DefaultSolrCoreState.java:326)
         at 
org.apache.solr.cloud.ZkController.checkRecovery(ZkController.java:1095)        
 at org.apache.solr.cloud.ZkController.register(ZkController.java:929)         
at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   90) Thread[id=17682, 
name=searcherExecutor-4390-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   91) Thread[id=18122, 
name=coreZkRegister-3197-thread-12-processing-n:127.0.0.1:33469__wgo 
x:awholynewstresscollection_collection2_4_shard7_replica2 s:shard7 
c:awholynewstresscollection_collection2_4 r:core_node12, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1029)       
  at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:999)   
      at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:982)    
     at org.apache.solr.cloud.ZkController.register(ZkController.java:896)      
   at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   92) Thread[id=19435, 
name=searcherExecutor-4558-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   93) Thread[id=19695, 
name=zkCallback-873-thread-75, state=TIMED_WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   94) Thread[id=17720, 
name=searcherExecutor-4421-thread-1, state=WAITING, 
group=TGRP-CollectionsAPIDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Thread

[...truncated too long message...]

.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   240) Thread[id=18163, name=coreZkRegister-3197-thread-21, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   241) Thread[id=18192, name=coreZkRegister-3186-thread-13, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   242) Thread[id=16139, name=Scheduler-1744411394, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   243) Thread[id=19584, name=searcherExecutor-4664-thread-1, 
state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   244) Thread[id=19449, name=searcherExecutor-4566-thread-1, 
state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   245) Thread[id=17715, name=searcherExecutor-4420-thread-1, 
state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   246) Thread[id=15907, name=Scheduler-23691200, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   247) Thread[id=19635, name=zkCallback-870-thread-63, 
state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([5E4A046F50D6BFE0]:0)
   [junit4] Completed [263/512] on J1 in 1186.58s, 1 test, 3 errors <<< 
FAILURES!

[...truncated 762 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:533:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:474:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:61:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/extra-targets.xml:39:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:230:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:513:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1426:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:981:
 There were test failures: 512 suites, 2014 tests, 2 suite-level errors, 2 
errors, 888 ignored (9 assumptions)

Total time: 168 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No prior successful build to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org