Build: https://ci-builds.apache.org/job/Solr/job/Solr-Check-9.x/4398/

5 tests failed.
FAILED:  org.apache.solr.TestDistributedGrouping.test

Error Message:
java.lang.Exception: Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)


FAILED:  org.apache.solr.TestDistributedGrouping.classMethod

Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)


FAILED:  org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown

Error Message:
java.lang.AssertionError: Timeout waiting for number of live nodes = 0

Stack Trace:
java.lang.AssertionError: Timeout waiting for number of live nodes = 0
        at 
__randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE:1DD98CD08D6F563D]:0)
        at org.junit.Assert.fail(Assert.java:89)
        at 
org.apache.solr.cloud.ZkFailoverTest.waitForLiveNodes(ZkFailoverTest.java:110)
        at 
org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown(ZkFailoverTest.java:63)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.ZkFailoverTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from 
SUITE scope at org.apache.solr.cloud.ZkFailoverTest: 
   1) Thread[id=258, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=261, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-EventThread,
 state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
   3) Thread[id=259, name=zkConnectionManagerCallback-133-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=257, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=260, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=RUNNABLE, group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
        at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
   6) Thread[id=248, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   7) Thread[id=254, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=262, name=zkConnectionManagerCallback-135-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=247, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
  10) Thread[id=249, 
name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:38451, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
  11) Thread[id=252, name=ProcessThread(sid:0 cport:38451):, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
  12) Thread[id=255, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=RUNNABLE, group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
        at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
  13) Thread[id=256, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-EventThread,
 state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
  14) Thread[id=251, name=SyncThread:0, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
  15) Thread[id=245, name=ZkTestServer Run Thread, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at [email protected]/java.lang.Thread.join(Thread.java:1300)
        at [email protected]/java.lang.Thread.join(Thread.java:1375)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:915)
        at 
app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:337)
        at app//org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:581)
  16) Thread[id=253, name=RequestThrottler, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from 
SUITE scope at org.apache.solr.cloud.ZkFailoverTest: 
   1) Thread[id=258, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=261, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-EventThread,
 state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
   3) Thread[id=259, name=zkConnectionManagerCallback-133-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=257, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=260, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=RUNNABLE, group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
        at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
   6) Thread[id=248, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   7) Thread[id=254, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=262, name=zkConnectionManagerCallback-135-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=247, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
  10) Thread[id=249, 
name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:38451, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
  11) Thread[id=252, name=ProcessThread(sid:0 cport:38451):, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
  12) Thread[id=255, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=RUNNABLE, group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136)
        at 
app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
  13) Thread[id=256, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-EventThread,
 state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
  14) Thread[id=251, name=SyncThread:0, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
  15) Thread[id=245, name=ZkTestServer Run Thread, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Object.wait(Native Method)
        at [email protected]/java.lang.Thread.join(Thread.java:1300)
        at [email protected]/java.lang.Thread.join(Thread.java:1375)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:915)
        at 
app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:337)
        at app//org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:581)
  16) Thread[id=253, name=RequestThrottler, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)


FAILED:  org.apache.solr.cloud.ZkFailoverTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=258, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=247, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.IOUtil.write1(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:254)
        at 
[email protected]/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at 
[email protected]/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at [email protected]/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:94)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:116)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   3) Thread[id=249, 
name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:38451, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
   4) Thread[id=259, name=zkConnectionManagerCallback-133-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=255, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=TIMED_WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1204)
   6) Thread[id=257, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=260, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=TIMED_WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1204)
   8) Thread[id=248, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.IOUtil.drain(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:266)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:203)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:137)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   9) Thread[id=254, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=262, name=zkConnectionManagerCallback-135-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=258, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=247, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.IOUtil.write1(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:254)
        at 
[email protected]/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at 
[email protected]/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at [email protected]/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:94)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:116)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   3) Thread[id=249, 
name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:38451, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
   4) Thread[id=259, name=zkConnectionManagerCallback-133-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=255, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=TIMED_WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1204)
   6) Thread[id=257, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=260, 
name=TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[734E0F9CF488F0BE]-SendThread(127.0.0.1:38451),
 state=TIMED_WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/java.lang.Thread.sleep(Native Method)
        at 
app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at 
app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1204)
   8) Thread[id=248, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, 
group=TGRP-ZkFailoverTest]
        at [email protected]/sun.nio.ch.IOUtil.drain(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:266)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:203)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:137)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   9) Thread[id=254, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=262, name=zkConnectionManagerCallback-135-thread-1, 
state=WAITING, group=TGRP-ZkFailoverTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:829)
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)




Build Log:
[...truncated 2593 lines...]
# ERROR: Error from server at 
http://localhost:8983/solr/admin/collections?action=CREATE&name=COLL_NAME&collection.configName=COLL_NAME&numShards=1&nrtReplicas=1&wt=javabin&version=2:
 org.apache.solr.cluster.placement.PlacementException: Not enough eligible 
nodes to place 1 replica(s) of type NRT for shard shard1 of collection COLL_NAME
ok 40 Random placement plugin using ENV in 22311ms
ok 41 post solr log into solr in 9496ms
ok 42 start solr with ssl in 19791ms
ok 43 start solr with ssl and auth in 22593ms
ok 44 SOLR-11740 check 'solr stop' connection in 28090ms
ok 45 status detects locally running solr in 14376ms
ok 46 status does not expose cli parameters to end user in 207ms
ok 47 listing out files in 2156ms
ok 48 copying files around in 6606ms
ok 49 upconfig in 2597ms

> Task :solr:packaging:integrationTests FAILED

> Task :solr:core:test

org.apache.solr.TestDistributedGrouping > test FAILED
    java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)

org.apache.solr.TestDistributedGrouping > classMethod FAILED
    java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)

org.apache.solr.TestDistributedGrouping > test suite's output saved to 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.TestDistributedGrouping.txt,
 copied below:
  2> 3145 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to 
test-framework derived value of 
'/tmp/src/solr/solr/server/solr/configsets/_default/conf'
  2> 3158 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
  2> 3187 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-1 for ERROR logs matching 
regex: ignore_exception
  2> 3207 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.SolrTestCaseJ4 Created dataDir: 
/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/data-dir-1-001
  2> 3322 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
  2> 3360 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, 
clientAuth=0.0/0.0)
  2> 3396 INFO  (SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]-worker) 
[] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: 
/yn/sv
  2> 3416 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.u.ErrorLogMuter Closing ErrorLogMuter-regex-1 after mutting 0 log messages
  2> 3417 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.u.ErrorLogMuter Creating ErrorLogMuter-regex-2 for ERROR logs matching 
regex: ignore_exception
  2> 3799 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/cores/collection1
  2> 4240 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
  2> 4243 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
  2> 4251 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.e.j.s.Server jetty-10.0.14; built: 2023-02-22T23:12:32.272Z; git: 
976721d0f3e903a243584d47870ad2f2c1bf9e55; jvm 11.0.16+8
  2> 4282 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.e.j.s.s.DefaultSessionIdManager Session workerName=node0
  2> 4311 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@332765c2{/yn/sv,file:///tmp/src/solr/solr/core/build/tmp/tests-cwd/,AVAILABLE}
  2> 4333 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.e.j.s.AbstractConnector Started ServerConnector@6ffc827e{HTTP/1.1, (http/1.1, 
h2c)}{127.0.0.1:43759}
  2> 4338 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.e.j.s.Server Started Server@44b12281{STARTING}[10.0.14,sto=0] @4354ms
  2> 4351 ERROR (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
  2> 4351 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
  2> 4358 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider  ___      _       Welcome to Apache Solr™ version 
9.3.0
  2> 4360 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider / __| ___| |_ _   Starting in standalone mode on 
port null
  2> 4360 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider \__ \/ _ \ | '_|  Install dir: /tmp/src/solr/solr
  2> 4361 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider |___/\___/_|_|    Start time: 
2023-05-23T07:00:55.288179Z
  2> 4362 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.s.CoreContainerProvider Solr Home: 
/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control
 (source: servlet config: solr.solr.home)
  2> 4364 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.c.SolrXmlConfig Loading solr.xml from 
/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/solr.xml
  2> 6081 WARN  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.c.CoreContainer Not all security plugins configured!  
authentication=disabled authorization=disabled.  Solr is only as secure as you 
make it. Consider configuring authentication/authorization before exposing Solr 
to users internal or external.  See https://s.apache.org/solrsecurity for more 
info
  2> 6353 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/cores
  2> 6354 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
  2> 6652 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.SolrResourceLoader Added 2 libs to classloader, from paths: 
[/tmp/src/solr/solr/core/build/resources/test/solr/collection1/lib, 
/tmp/src/solr/solr/core/build/resources/test/solr/collection1/lib/classes]
  2> 7011 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.SolrConfig You should not use LATEST as luceneMatchVersion property: if 
you use this setting, and then Solr upgrades to a newer release of Lucene, 
sizable changes may happen. If precise back compatibility is important then you 
should instead explicitly specify an actual Lucene version.
  2> 7012 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.5.0
  2> 7025 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.SolrConfig solrconfig.xml: <boolTofilterOptimizer> is currently not 
implemented and has no effect.
  2> 7101 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.s.IndexSchema Schema name=test
  2> 7119 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.TrieIntField Solr loaded a deprecated plugin/analysis class 
[solr.TrieIntField]. Please consult documentation how to replace it accordingly.
  2> 7125 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.TrieFloatField Solr loaded a deprecated plugin/analysis class 
[solr.TrieFloatField]. Please consult documentation how to replace it 
accordingly.
  2> 7129 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.TrieLongField Solr loaded a deprecated plugin/analysis class 
[solr.TrieLongField]. Please consult documentation how to replace it 
accordingly.
  2> 7132 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.TrieDoubleField Solr loaded a deprecated plugin/analysis class 
[solr.TrieDoubleField]. Please consult documentation how to replace it 
accordingly.
  2> 7209 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.TrieDateField Solr loaded a deprecated plugin/analysis class 
[solr.TrieDateField]. Please consult documentation how to replace it 
accordingly.
  2> 7270 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.WordDelimiterFilterFactory Solr loaded a deprecated plugin/analysis 
class [solr.WordDelimiterFilterFactory]. Please consult documentation how to 
replace it accordingly.
  2> 7293 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.CurrencyField Solr loaded a deprecated plugin/analysis class 
[solr.CurrencyField]. Please consult documentation how to replace it 
accordingly.
  2> 7305 WARN  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.D.s.EnumField Solr loaded a deprecated plugin/analysis class 
[solr.EnumField]. Please consult documentation how to replace it accordingly.
  2> 7571 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
  2> 7636 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
  2> 7771 INFO  (TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]) [] 
o.a.s.e.JettySolrRunner Jetty properties: {hostContext=/yn/sv, hostPort=43759, 
coreRootDirectory=/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/cores}
  2> 7919 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from 
configset /tmp/src/solr/solr/core/build/resources/test/solr/collection1, 
trusted=true
  2> 7992 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.c.SolrCore Opening new SolrCore at 
[/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/cores/collection1],
 
dataDir=[/tmp/src/solr/solr/core/build/tmp/tests-tmp/solr.TestDistributedGrouping_734E0F9CF488F0BE-001/tempDir-001/control/cores/collection1/data/]
  2> 8038 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022208E12, 
noCFSRatio=0.5267999975881604]
  2> 8927 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.j.SolrRequestAuthorizer Creating a new SolrRequestAuthorizer
  2> 8980 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
  2> 8980 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
  2> 9000 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
  2> 9001 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
  2> 9008 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=30, maxMergedSegmentMB=37.80839443206787, 
floorSegmentMB=0.8587284088134766, 
forceMergeDeletesPctAllowed=4.388833933585037, segmentsPerTier=31.0, 
maxCFSSegmentSizeMB=8.796093022208E12, noCFSRatio=0.0, 
deletesPctAllowed=38.54343639490358
  2> 9195 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.s.DirectSolrSpellChecker init: {name=direct, 
classname=DirectSolrSpellChecker, field=lowerfilt, minQueryLength=3}
  2> 9196 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.s.DirectSolrSpellChecker init: {name=directMQF2, 
classname=DirectSolrSpellChecker, field=lowerfilt, minQueryLength=3, 
maxQueryFrequency=2}
  2> 9352 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
  2> 9361 INFO  (coreLoadExecutor-12-thread-1) [ x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1766667332131749888
  2> 9372 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: default
  2> 9375 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: direct
  2> 9376 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: directMQF2
  2> 9376 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: wordbreak
  2> 9376 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: multipleFields
  2> 9379 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: jarowinkler
  2> 9381 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: external
  2> 9386 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: freq
  2> 9389 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: fqcn
  2> 9391 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for 
spellchecker: perDict
  2> 9407 INFO  (searcherExecutor-14-thread-1-processing-collection1) [ 
x:collection1] o.a.s.c.SolrCore Registered new searcher autowarm time: 0 ms
  2> મે 23, 2023 5:00:53 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
  2> WARNING: Suite execution timed out: org.apache.solr.TestDistributedGrouping
  2> ==== jstack at approximately timeout time ====
  2> "Test worker" ID=1 WAITING on 
com.carrotsearch.randomizedtesting.RandomizedRunner$2@18de64e2
  2>    at [email protected]/java.lang.Object.wait(Native Method)
  2>    - waiting on 
com.carrotsearch.randomizedtesting.RandomizedRunner$2@18de64e2
  2>    at [email protected]/java.lang.Thread.join(Thread.java:1300)
  2>    at [email protected]/java.lang.Thread.join(Thread.java:1375)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:647)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:504)
  2>    at 
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:108)
  2>    at 
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:57)
  2>    at 
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:39)
  2>    at 
org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:62)
  2>    at 
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:52)
  2>    at 
[email protected]/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
  2>    at 
[email protected]/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  2>    at 
[email protected]/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  2>    at [email protected]/java.lang.reflect.Method.invoke(Method.java:566)
  2>    at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
  2>    at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
  2>    at 
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
  2>    at 
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
  2>    at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker$2.run(TestWorker.java:176)
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129)
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100)
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60)
  2>    at 
org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56)
  2>    at 
org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113)
  2>    at 
org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65)
  2>    at 
app//worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69)
  2>    at 
app//worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74)
  2> 
  2> "Reference Handler" ID=2 RUNNABLE
  2>    at 
[email protected]/java.lang.ref.Reference.waitForReferencePendingList(Native 
Method)
  2>    at 
[email protected]/java.lang.ref.Reference.processPendingReferences(Reference.java:241)
  2>    at 
[email protected]/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:213)
  2> 
  2> "Finalizer" ID=3 WAITING on java.lang.ref.ReferenceQueue$Lock@483b5100
  2>    at [email protected]/java.lang.Object.wait(Native Method)
  2>    - waiting on java.lang.ref.ReferenceQueue$Lock@483b5100
  2>    at 
[email protected]/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
  2>    at 
[email protected]/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
  2>    at 
[email protected]/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:170)
  2> 
  2> "Signal Dispatcher" ID=4 RUNNABLE
  2> 
  2> "Common-Cleaner" ID=9 TIMED_WAITING on 
java.lang.ref.ReferenceQueue$Lock@67c09854
  2>    at [email protected]/java.lang.Object.wait(Native Method)
  2>    - timed waiting on java.lang.ref.ReferenceQueue$Lock@67c09854
  2>    at 
[email protected]/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
  2>    at 
[email protected]/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    at 
[email protected]/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:161)
  2> 
  2> "/127.0.0.1:41692 to /127.0.0.1:35943 workers" ID=11 WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b3beeee
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b3beeee
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
  2>    at 
[email protected]/java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:367)
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker.submitToRun(TestWorker.java:207)
  2>    - locked 
org.gradle.api.internal.tasks.testing.worker.TestWorker@68427249
  2>    at 
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:169)
  2>    at 
[email protected]/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
  2>    at 
[email protected]/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  2>    at 
[email protected]/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  2>    at [email protected]/java.lang.reflect.Method.invoke(Method.java:566)
  2>    at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
  2>    at 
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
  2>    at 
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
  2>    at 
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
  2>    at 
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414)
  2>    at 
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
  2>    at 
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:49)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    Locked synchronizers:
  2>    - java.util.concurrent.ThreadPoolExecutor$Worker@294425a7
  2> 
  2> "/127.0.0.1:41692 to /127.0.0.1:35943 workers Thread 2" ID=12 WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@91a0fd8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@91a0fd8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
  2>    at 
org.gradle.internal.remote.internal.hub.queue.EndPointQueue.take(EndPointQueue.java:49)
  2>    at 
org.gradle.internal.remote.internal.hub.MessageHub$ConnectionDispatch.run(MessageHub.java:322)
  2>    at 
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
  2>    at 
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:49)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    Locked synchronizers:
  2>    - java.util.concurrent.ThreadPoolExecutor$Worker@3270d194
  2> 
  2> "/127.0.0.1:41692 to /127.0.0.1:35943 workers Thread 3" ID=13 RUNNABLE (in 
native code)
  2>    at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
  2>    at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
  2>    at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
  2>    - locked sun.nio.ch.Util$2@6a7b2497
  2>    - locked sun.nio.ch.EPollSelectorImpl@1150d3ab
  2>    at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
  2>    at 
org.gradle.internal.remote.internal.inet.SocketConnection$SocketInputStream.read(SocketConnection.java:185)
  2>    at com.esotericsoftware.kryo.io.Input.fill(Input.java:146)
  2>    at com.esotericsoftware.kryo.io.Input.require(Input.java:178)
  2>    at com.esotericsoftware.kryo.io.Input.readByte(Input.java:295)
  2>    at 
org.gradle.internal.serialize.kryo.KryoBackedDecoder.readByte(KryoBackedDecoder.java:88)
  2>    at 
org.gradle.internal.remote.internal.hub.InterHubMessageSerializer$MessageReader.read(InterHubMessageSerializer.java:64)
  2>    at 
org.gradle.internal.remote.internal.hub.InterHubMessageSerializer$MessageReader.read(InterHubMessageSerializer.java:52)
  2>    at 
org.gradle.internal.remote.internal.inet.SocketConnection.receive(SocketConnection.java:81)
  2>    at 
org.gradle.internal.remote.internal.hub.MessageHub$ConnectionReceive.run(MessageHub.java:270)
  2>    at 
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
  2>    at 
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:49)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    Locked synchronizers:
  2>    - java.util.concurrent.ThreadPoolExecutor$Worker@76505305
  2> 
  2> "SUITE-TestDistributedGrouping-seed#[734E0F9CF488F0BE]" ID=14 RUNNABLE
  2>    at 
[email protected]/sun.management.ThreadImpl.dumpThreads0(Native Method)
  2>    at 
[email protected]/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:521)
  2>    at 
[email protected]/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:509)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:708)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:59)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:437)
  2>    - locked java.lang.Object@55d803f1
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:716)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:637)
  2> 
  2> "TEST-TestDistributedGrouping.test-seed#[734E0F9CF488F0BE]" ID=15 WAITING 
on java.util.concurrent.CountDownLatch$Sync@68acfebc
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - waiting on java.util.concurrent.CountDownLatch$Sync@68acfebc
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
  2>    at 
[email protected]/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
  2>    at 
app//org.apache.solr.servlet.CoreContainerProvider$ContextInitializationKey.waitForReadyService(CoreContainerProvider.java:510)
  2>    at 
app//org.apache.solr.servlet.CoreContainerProvider$ServiceHolder.getService(CoreContainerProvider.java:549)
  2>    at 
app//org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:147)
  2>    at 
app//org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:133)
  2>    at 
app//org.eclipse.jetty.servlet.ServletHandler.initializeHolders(ServletHandler.java:774)
  2>    at 
app//org.eclipse.jetty.servlet.ServletHandler.setFilters(ServletHandler.java:1472)
  2>    at 
app//org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:992)
  2>    at 
app//org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:480)
  2>    at 
app//org.apache.solr.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:427)
  2>    - locked org.apache.solr.embedded.JettySolrRunner$1@ce5b25d
  2>    at 
app//org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:253)
  2>    at 
app//org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:94)
  2>    at 
app//org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:616)
  2>    at 
app//org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:554)
  2>    - locked org.apache.solr.embedded.JettySolrRunner@3f8960ca
  2>    at 
app//org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:525)
  2>    at 
app//org.apache.solr.BaseDistributedSearchTestCase.createControlJetty(BaseDistributedSearchTestCase.java:362)
  2>    at 
app//org.apache.solr.BaseDistributedSearchTestCase.createServers(BaseDistributedSearchTestCase.java:370)
  2>    at 
app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1161)
  2>    at 
app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1134)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
  2>    at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
  2>    at 
app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  2>    at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
  2>    at 
app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  2>    at 
app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
  2>    at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
  2>    at 
app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
  2>    at 
app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  2>    at 
app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
  2>    at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
  2>    at 
app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
  2>    at 
app//com.carrotsearch.randomizedtesting.ThreadLeakControl$$Lambda$186/0x0000000840222c40.run(Unknown
 Source)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    Locked synchronizers:
  2>    - java.util.concurrent.locks.ReentrantLock$NonfairSync@28d3311b
  2>    - java.util.concurrent.locks.ReentrantLock$NonfairSync@3d01787b
  2>    - java.util.concurrent.locks.ReentrantLock$NonfairSync@1cd752dc
  2> 
  2> "qtp1637886620-17" ID=17 RUNNABLE (in native code)
  2>    at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
  2>    at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
  2>    at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
  2>    - locked sun.nio.ch.Util$2@5946415b
  2>    - locked sun.nio.ch.EPollSelectorImpl@6fb238a8
  2>    at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
  2>    at 
app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:180)
  2>    at 
app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:187)
  2>    at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:604)
  2>    at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:541)
  2>    at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:450)
  2>    at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:243)
  2>    at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:194)
  2>    at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$260/0x0000000840335440.run(Unknown
 Source)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-18-acceptor-0@66b687f4-ServerConnector@6ffc827e{HTTP/1.1, 
(http/1.1, h2c)}{127.0.0.1:43759}" ID=18 RUNNABLE (in native code)
  2>    at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept0(Native 
Method)
  2>    at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
  2>    at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
  2>    at 
app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:409)
  2>    at 
app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:748)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>    Locked synchronizers:
  2>    - java.util.concurrent.locks.ReentrantLock$NonfairSync@49d5d630
  2> 
  2> "qtp1637886620-19" ID=19 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-20" ID=20 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-21" ID=21 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-22" ID=22 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-23" ID=23 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "qtp1637886620-24" ID=24 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@468388a8
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>    at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "Session-HouseKeeper-6ea11afe-1" ID=25 TIMED_WAITING on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2d2445ba
  2>    at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
  2>    - timed waiting on 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2d2445ba
  2>    at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>    at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>    at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
  2>    at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  2>    at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>    at [email protected]/java.lang.Thread.run(Thread.java:829)
  2> 
  2> "httpShardExecutor-7-thread-1" ID=26 RUNNABLE (in native code)
  2>    at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
  2>    at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
  2>    at [email protected]/sun.nio.ch.Se

[...truncated too long message...]

>         at 
> [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
  2>         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2>         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:219)
  2>         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:1017)
  2>         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1067)
  2>         at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>   14) Thread[id=31, name=h2sc-9-thread-2, state=RUNNABLE, 
group=TGRP-TestDistributedGrouping]
  2>         at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
  2>         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
  2>         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
  2>         at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:180)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:187)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:604)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:541)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:450)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:243)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:194)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$260/0x0000000840335440.run(Unknown
 Source)
  2>         at 
app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
  2>         at 
app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$347/0x000000084041d840.run(Unknown
 Source)
  2>         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2>         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>         at [email protected]/java.lang.Thread.run(Thread.java:829)
  2>   15) Thread[id=27, name=httpShardExecutor-7-thread-2, state=RUNNABLE, 
group=TGRP-TestDistributedGrouping]
  2>         at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
  2>         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
  2>         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
  2>         at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:180)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:187)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:604)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:541)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:450)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:243)
  2>         at 
app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:194)
  2>         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$260/0x0000000840335440.run(Unknown
 Source)
  2>         at 
app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
  2>         at 
app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$347/0x000000084041d840.run(Unknown
 Source)
  2>         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2>         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2>         at [email protected]/java.lang.Thread.run(Thread.java:829)
   >     java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   >         at __randomizedtesting.SeedInfo.seed([734E0F9CF488F0BE]:0)
  2> NOTE: test params are: codec=CheapBastard, 
sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=gu, 
timezone=America/Tortola
  2> NOTE: Linux 5.4.0-100-generic amd64/Oracle Corporation 11.0.16 
(64-bit)/cpus=1,threads=18,free=198693512,total=324534272
  2> NOTE: All tests run in this JVM: [TestDistributedGrouping]
  2> NOTE: reproduce with: gradlew test --tests TestDistributedGrouping 
-Dtests.seed=734E0F9CF488F0BE -Dtests.locale=gu 
-Dtests.timezone=America/Tortola -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

:solr:core:test (FAILURE): 4743 test(s), 5 failure(s), 251 skipped

4743 tests completed, 5 failed, 251 skipped

> Task :solr:core:test FAILED
> Task :solr:core:wipeTaskTemp

ERROR: The following test(s) have failed:
  - org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown 
(:solr:core)
    Test output: 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ZkFailoverTest.txt
    Reproduce with: gradlew :solr:core:test --tests 
"org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown" 
-Ptests.jvms=96 "-Ptests.jvmargs=-XX:TieredStopAtLevel=1 -XX:+UseParallelGC 
-XX:ActiveProcessorCount=1 -XX:ReservedCodeCacheSize=120m" 
-Ptests.seed=734E0F9CF488F0BE -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.ZkFailoverTest.classMethod (:solr:core)
    Test output: 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ZkFailoverTest.txt
    Reproduce with: gradlew :solr:core:test --tests 
"org.apache.solr.cloud.ZkFailoverTest.classMethod" -Ptests.jvms=96 
"-Ptests.jvmargs=-XX:TieredStopAtLevel=1 -XX:+UseParallelGC 
-XX:ActiveProcessorCount=1 -XX:ReservedCodeCacheSize=120m" 
-Ptests.seed=734E0F9CF488F0BE -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.ZkFailoverTest.classMethod (:solr:core)
    Test output: 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ZkFailoverTest.txt
    Reproduce with: gradlew :solr:core:test --tests 
"org.apache.solr.cloud.ZkFailoverTest.classMethod" -Ptests.jvms=96 
"-Ptests.jvmargs=-XX:TieredStopAtLevel=1 -XX:+UseParallelGC 
-XX:ActiveProcessorCount=1 -XX:ReservedCodeCacheSize=120m" 
-Ptests.seed=734E0F9CF488F0BE -Ptests.file.encoding=US-ASCII

  - org.apache.solr.TestDistributedGrouping.test (:solr:core)
    Test output: 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.TestDistributedGrouping.txt
    Reproduce with: gradlew :solr:core:test --tests 
"org.apache.solr.TestDistributedGrouping.test" -Ptests.jvms=96 
"-Ptests.jvmargs=-XX:TieredStopAtLevel=1 -XX:+UseParallelGC 
-XX:ActiveProcessorCount=1 -XX:ReservedCodeCacheSize=120m" 
-Ptests.seed=734E0F9CF488F0BE -Ptests.file.encoding=US-ASCII

  - org.apache.solr.TestDistributedGrouping.classMethod (:solr:core)
    Test output: 
/tmp/src/solr/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.TestDistributedGrouping.txt
    Reproduce with: gradlew :solr:core:test --tests 
"org.apache.solr.TestDistributedGrouping.classMethod" -Ptests.jvms=96 
"-Ptests.jvmargs=-XX:TieredStopAtLevel=1 -XX:+UseParallelGC 
-XX:ActiveProcessorCount=1 -XX:ReservedCodeCacheSize=120m" 
-Ptests.seed=734E0F9CF488F0BE -Ptests.file.encoding=US-ASCII


FAILURE: Build completed with 2 failures.

1: Task failed with an exception.
-----------
* Where:
Build file '/tmp/src/solr/solr/packaging/build.gradle' line: 285

* What went wrong:
Execution failed for task ':solr:packaging:integrationTests'.
> Process 'command 
> '/tmp/src/solr/.gradle/node/packaging/node_modules/bats/bin/bats'' finished 
> with non-zero exit value 1

* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
==============================================================================

2: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':solr:core:test'.
> There were failing tests. See the results at: 
> file:///tmp/src/solr/solr/core/build/test-results/test/

* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
==============================================================================

* Get more help at https://help.gradle.org

BUILD FAILED in 2h 3m 45s
625 actionable tasks: 609 executed, 16 up-to-date
[?12l[?25h
------------------------------------------------------------------------
Build Failed: returned 1

Total time: 2h4m8s
------------------------------------------------------------------------
+ status=1
+ ./crave pull '**/build/**/test/TEST-*.xml' '**/*.events' 'heapdumps/**' 
'**/hs_err_pid*'
Error: rsync: [sender] change_dir "/tmp/src/solr/heapdumps" failed: No 
such file or directory (2)
rsync error: some files/attrs were not transferred (see previous errors) (code 
23) at main.c(1675) [Receiver=3.1.2]
rsync: [Receiver] write error: Broken pipe (32)

+ exit 1
Build step 'Execute shell' marked build as failure
Archiving artifacts
Recording test results
[Checks API] No suitable checks publisher found.
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to