Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.x-MacOSX/1399/ Java: 64bit/jdk-14.0.1 -XX:-UseCompressedOops -XX:+UseSerialGC
14 tests failed. FAILED: org.apache.solr.TestDistributedGrouping.test Error Message: Error from server at http://127.0.0.1:49680/collection1: org.apache.solr.client.solrj.SolrServerException: Time allowed to handle this request exceeded:[http://[::1]:4, http://[::1]:8, http://[::1]:6] Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49680/collection1: org.apache.solr.client.solrj.SolrServerException: Time allowed to handle this request exceeded:[http://[::1]:4, http://[::1]:8, http://[::1]:6] at __randomizedtesting.SeedInfo.seed([8D956CADB91B6592:5C1537717E7086A]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1003) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1018) at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:625) at org.apache.solr.TestDistributedGrouping.simpleQuery(TestDistributedGrouping.java:455) at org.apache.solr.TestDistributedGrouping.test(TestDistributedGrouping.java:443) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1117) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:832) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest Error Message: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest: 1) Thread[id=9809, name=SessionTracker, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at [email protected]/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:157) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest: 1) Thread[id=9809, name=SessionTracker, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at [email protected]/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:157) at __randomizedtesting.SeedInfo.seed([8D956CADB91B6592]:0) FAILED: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test Error Message: Number of replicas in the state does not match what we set:2 vs 3 Stack Trace: org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:2 vs 3 at __randomizedtesting.SeedInfo.seed([8D956CADB91B6592:5C1537717E7086A]:0) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:633) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:572) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:358) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1088) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:832) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailoverAfterPartitionTest Error Message: 45 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest: 1) Thread[id=10302, name=SolrRrdBackendFactory-7148-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 2) Thread[id=10223, name=Connector-Scheduler-58c83ce9-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 3) Thread[id=10120, name=qtp1107170144-10120-acceptor-0@49651222-ServerConnector@45c19f87{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53849}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.accept(Native Method) at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 4) Thread[id=10149, name=qtp1559737263-10149, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 5) Thread[id=10174, name=qtp2080926231-10174, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 6) Thread[id=10177, name=qtp2080926231-10177-acceptor-0@53e82b30-ServerConnector@45ed4c28{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53871}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.accept(Native Method) at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 7) Thread[id=10180, name=qtp2080926231-10180, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 8) Thread[id=10143, name=Thread-1517, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 9) Thread[id=10186, name=Thread-1534, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 10) Thread[id=10181, name=qtp2080926231-10181, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 11) Thread[id=10299, name=SolrRrdBackendFactory-7104-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 12) Thread[id=10154, name=qtp1559737263-10154, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 13) Thread[id=10146, name=SocketProxy-Acceptor-53848, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.poll(Native Method) at [email protected]/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181) at [email protected]/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710) at [email protected]/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:684) at [email protected]/java.net.ServerSocket.platformImplAccept(ServerSocket.java:650) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:626) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:583) at [email protected]/java.net.ServerSocket.accept(ServerSocket.java:540) at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:447) at [email protected]/java.lang.Thread.run(Thread.java:832) 14) Thread[id=10156, name=qtp1559737263-10156, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 15) Thread[id=10204, name=Thread-1535, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 16) Thread[id=10119, name=qtp1107170144-10119, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 17) Thread[id=10188, name=SocketProxy-Acceptor-53856, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.poll(Native Method) at [email protected]/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181) at [email protected]/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710) at [email protected]/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:684) at [email protected]/java.net.ServerSocket.platformImplAccept(ServerSocket.java:650) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:626) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:583) at [email protected]/java.net.ServerSocket.accept(ServerSocket.java:540) at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:447) at [email protected]/java.lang.Thread.run(Thread.java:832) 18) Thread[id=10141, name=SolrRrdBackendFactory-7104-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 19) Thread[id=10300, name=SolrRrdBackendFactory-7126-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 20) Thread[id=10142, name=MetricsHistoryHandler-7105-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 21) Thread[id=10157, name=Session-HouseKeeper-25fe3221-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 22) Thread[id=10202, name=SolrRrdBackendFactory-7148-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 23) Thread[id=10235, name=Connector-Scheduler-45ed4c28-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 24) Thread[id=10124, name=qtp1107170144-10124, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 25) Thread[id=10122, name=qtp1107170144-10122, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 26) Thread[id=10123, name=qtp1107170144-10123, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 27) Thread[id=10203, name=MetricsHistoryHandler-7149-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 28) Thread[id=10182, name=Session-HouseKeeper-745e3a81-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 29) Thread[id=10152, name=qtp1559737263-10152, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 30) Thread[id=10172, name=SolrRrdBackendFactory-7126-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 31) Thread[id=10125, name=Session-HouseKeeper-4658d262-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 32) Thread[id=10175, name=qtp2080926231-10175, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 33) Thread[id=10121, name=qtp1107170144-10121, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 34) Thread[id=10150, name=qtp1559737263-10150-acceptor-0@362290ba-ServerConnector@58c83ce9{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53858}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.accept(Native Method) at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 35) Thread[id=10117, name=qtp1107170144-10117, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 36) Thread[id=10155, name=qtp1559737263-10155, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 37) Thread[id=10178, name=qtp2080926231-10178, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 38) Thread[id=10206, name=SocketProxy-Acceptor-53870, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.poll(Native Method) at [email protected]/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181) at [email protected]/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710) at [email protected]/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:684) at [email protected]/java.net.ServerSocket.platformImplAccept(ServerSocket.java:650) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:626) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:583) at [email protected]/java.net.ServerSocket.accept(ServerSocket.java:540) at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:447) at [email protected]/java.lang.Thread.run(Thread.java:832) 39) Thread[id=10118, name=qtp1107170144-10118, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 40) Thread[id=10173, name=MetricsHistoryHandler-7127-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:126) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:81) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:87) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:55) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:325) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:360) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:355) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:456) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:382) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:237) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$438/0x0000000105dd3040.run(Unknown Source) at [email protected]/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at [email protected]/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 41) Thread[id=10147, name=qtp1559737263-10147, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 42) Thread[id=10176, name=qtp2080926231-10176, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 43) Thread[id=10179, name=qtp2080926231-10179, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 44) Thread[id=10220, name=Connector-Scheduler-45c19f87-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 45) Thread[id=10148, name=qtp1559737263-10148, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 45 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest: 1) Thread[id=10302, name=SolrRrdBackendFactory-7148-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 2) Thread[id=10223, name=Connector-Scheduler-58c83ce9-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 3) Thread[id=10120, name=qtp1107170144-10120-acceptor-0@49651222-ServerConnector@45c19f87{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53849}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.accept(Native Method) at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 4) Thread[id=10149, name=qtp1559737263-10149, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 5) Thread[id=10174, name=qtp2080926231-10174, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 6) Thread[id=10177, name=qtp2080926231-10177-acceptor-0@53e82b30-ServerConnector@45ed4c28{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53871}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.accept(Native Method) at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 7) Thread[id=10180, name=qtp2080926231-10180, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 8) Thread[id=10143, name=Thread-1517, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 9) Thread[id=10186, name=Thread-1534, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 10) Thread[id=10181, name=qtp2080926231-10181, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 11) Thread[id=10299, name=SolrRrdBackendFactory-7104-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 12) Thread[id=10154, name=qtp1559737263-10154, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 13) Thread[id=10146, name=SocketProxy-Acceptor-53848, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.poll(Native Method) at [email protected]/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181) at [email protected]/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710) at [email protected]/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:684) at [email protected]/java.net.ServerSocket.platformImplAccept(ServerSocket.java:650) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:626) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:583) at [email protected]/java.net.ServerSocket.accept(ServerSocket.java:540) at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:447) at [email protected]/java.lang.Thread.run(Thread.java:832) 14) Thread[id=10156, name=qtp1559737263-10156, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 15) Thread[id=10204, name=Thread-1535, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/java.lang.Object.wait(Native Method) at [email protected]/java.lang.Object.wait(Object.java:321) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2199) 16) Thread[id=10119, name=qtp1107170144-10119, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.KQueue.poll(Native Method) at [email protected]/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at [email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at [email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x00000001068b8058.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at [email protected]/java.lang.Thread.run(Thread.java:832) 17) Thread[id=10188, name=SocketProxy-Acceptor-53856, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/sun.nio.ch.Net.poll(Native Method) at [email protected]/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181) at [email protected]/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710) at [email protected]/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:684) at [email protected]/java.net.ServerSocket.platformImplAccept(ServerSocket.java:650) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:626) at [email protected]/java.net.ServerSocket.implAccept(ServerSocket.java:583) at [email protected]/java.net.ServerSocket.accept(ServerSocket.java:540) at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:447) at [email protected]/java.lang.Thread.run(Thread.java:832) 18) Thread[id=10141, name=SolrRrdBackendFactory-7104-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 19) Thread[id=10300, name=SolrRrdBackendFactory-7126-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 20) Thread[id=10142, name=MetricsHistoryHandler-7105-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 21) Thread[id=10157, name=Session-HouseKeeper-25fe3221-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 22) Thread[id=10202, name=SolrRrdBackendFactory-7148-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 23) Thread[id=10235, name=Connector-Scheduler-45ed4c28-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 24) Thread[id=10124, name=qtp1107170144-10124, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 25) Thread[id=10122, name=qtp1107170144-10122, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 26) Thread[id=10123, name=qtp1107170144-10123, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 27) Thread[id=10203, name=MetricsHistoryHandler-7149-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at [email protected]/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 28) Thread[id=10182, name=Session-HouseKeeper-745e3a81-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 29) Thread[id=10152, name=qtp1559737263-10152, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at [email protected]/java.lang.Thread.run(Thread.java:832) 30) Thread[id=10172, name=SolrRrdBackendFactory-7126-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at [email protected]/java.lang.Thread.run(Thread.java:832) 31) Thread[id=10125, name=Session-HouseKeeper-4658d262-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPo [...truncated too long message...] zip.GzipHandler.handle(GzipHandler.java:717) [junit4] > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) [junit4] > at org.eclipse.jetty.server.Server.handle(Server.java:500) [junit4] > at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) [junit4] > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) [junit4] > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) [junit4] > at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) [junit4] > at java.base/java.lang.Thread.run(Thread.java:832) [junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper [junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) [junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) [junit4] > at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98) [junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:816) [junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1007) [junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:924) [junit4] > at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1381) [junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1267) [junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) [junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) [junit4] > at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) [junit4] > at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) [junit4] > at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) [junit4] > at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854) [junit4] > at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818) [junit4] > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566) [junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) [junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) [junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) [junit4] > at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) [junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) [junit4] > at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) [junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) [junit4] > at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) [junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) [junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) [junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) [junit4] > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) [junit4] > at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) [junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) [junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) [junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [junit4] > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) [junit4] > at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) [junit4] > at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) [junit4] > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) [junit4] > at org.eclipse.jetty.server.Server.handle(Server.java:500) [junit4] > at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) [junit4] > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) [junit4] > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) [junit4] > at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) [junit4] > at java.base/java.lang.Thread.run(Thread.java:832) [junit4] > > [junit4] > at __randomizedtesting.SeedInfo.seed([8D956CADB91B6592]:0) [junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:326) [junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564) [junit4] > at java.base/java.lang.Thread.run(Thread.java:832) [junit4] Completed [34/35 (5!)] on J2 in 69.62s, 1 test, 1 failure <<< FAILURES! [...truncated 22 lines...] BUILD FAILED /Users/jenkins/workspace/Lucene-Solr-8.x-MacOSX/lucene/common-build.xml:1599: The following error occurred while executing this line: /Users/jenkins/workspace/Lucene-Solr-8.x-MacOSX/lucene/common-build.xml:1126: There were test failures: 35 suites, 85 tests, 4 suite-level errors, 1 error [seed: 8D956CADB91B6592] Total time: 4 minutes 23 seconds [repro] Setting last failure code to 256 [repro] Failures w/original seeds: [repro] 0/5 failed: org.apache.solr.TestDistributedGrouping [repro] 0/5 failed: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest [repro] 0/5 failed: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest [repro] 0/5 failed: org.apache.solr.cloud.ZkShardTermsTest [repro] 0/5 failed: org.apache.solr.security.BasicAuthIntegrationTest [repro] 1/5 failed: org.apache.solr.handler.TestContainerPlugin [repro] 4/5 failed: org.apache.solr.schema.SchemaApiFailureTest [repro] Exiting with code 256 + mv lucene/build lucene/build.repro + mv solr/build solr/build.repro + mv lucene/build.orig lucene/build + mv solr/build.orig solr/build Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Archiving artifacts [Java] Sleeping for 5 seconds due to JENKINS-32191... [Java] Parsing console log (workspace: '/Users/jenkins/workspace/Lucene-Solr-8.x-MacOSX') [Java] Successfully parsed console log [Java] -> found 112 issues (skipped 103 duplicates) [Java] Post processing issues on 'MacOSX VBOX' with source code encoding 'UTF-8' [Java] Creating SCM blamer to obtain author and commit information for affected files [Java] Creating SCM miner to obtain statistics for affected repository files [Java] Resolving file names for all issues in source directory '/Users/jenkins/workspace/Lucene-Solr-8.x-MacOSX' [Java] -> resolved paths in source directory (53 found, 0 not found) [Java] Resolving module names from module definitions (build.xml, pom.xml, or Manifest.mf files) [Java] -> resolved module names for 112 issues [Java] Resolving package names (or namespaces) by parsing the affected files [Java] -> resolved package names of 53 affected files [Java] No filter has been set, publishing all 112 issues [Java] Creating fingerprints for all affected code blocks to track issues over different builds [Java] -> created fingerprints for 112 issues (skipped 0 issues) [Java] Copying affected files to Jenkins' build folder '/home/jenkins/jobs/Lucene-Solr-8.x-MacOSX/builds/1399/files-with-issues' [Java] -> 53 copied, 0 not in workspace, 0 not-found, 0 with I/O error [Java] Using reference build 'Lucene-Solr-8.x-MacOSX #1398' to compute new, fixed, and outstanding issues [Java] Issues delta (vs. reference build): outstanding: 112, new: 0, fixed: 0 [Java] No quality gates have been set - skipping [Java] Health report is disabled - skipping [Java] Created analysis result for 112 issues (found 0 new issues, fixed 0 issues) [Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-8.x-MacOSX #1399'. Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Recording test results Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Build step 'Publish JUnit test result report' changed build result to UNSTABLE Email was triggered for: Unstable (Test Failures) Sending email for trigger: Unstable (Test Failures) Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2 Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
