Build: https://ci-builds.apache.org/job/Solr/job/Solr-NightlyTests-main/164/
12 tests failed. FAILED: org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34083/solr: Both 'node' and 'createNodeSet' parameters cannot be specified together. Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34083/solr: Both 'node' and 'createNodeSet' parameters cannot be specified together. at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:9BC51CEF082C2BE]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1159) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243) at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted(MoveReplicaHDFSFailoverTest.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition Error Message: java.lang.AssertionError Stack Trace: java.lang.AssertionError at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:4C723A653188EC7D]:0) at org.junit.Assert.fail(Assert.java:87) at org.junit.Assert.assertTrue(Assert.java:42) at org.junit.Assert.assertTrue(Assert.java:53) at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition(MoveReplicaHDFSFailoverTest.java:183) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained Error Message: java.lang.AssertionError Stack Trace: java.lang.AssertionError at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:691120C04DF83FFE]:0) at org.junit.Assert.fail(Assert.java:87) at org.junit.Assert.assertTrue(Assert.java:42) at org.junit.Assert.assertTrue(Assert.java:53) at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained(MoveReplicaHDFSFailoverTest.java:94) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36951/onenodecollectioncore: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36951/onenodecollectioncore: Error opening new searcher at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480) at org.apache.solr.cloud.BasicDistributedZk2Test.addAndQueryDocs(BasicDistributedZk2Test.java:187) at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:167) at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:76) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.classMethod Error Message: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=15969, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest] Stack Trace: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=15969, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-HdfsRecoveryZkTest] Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.index.CorruptIndexException: Problem reading index from NRTCachingDirectory(BlockDirectory(HdfsDirectory@hdfs://localhost:39347/solr_hdfs_home/recoverytest/core_node4/data/index lockFactory=org.apache.solr.store.hdfs.HdfsLockFactory@281fce99); maxCacheMB=192.0 maxMergeSizeMB=16.0) (resource=NRTCachingDirectory(BlockDirectory(HdfsDirectory@hdfs://localhost:39347/solr_hdfs_home/recoverytest/core_node4/data/index lockFactory=org.apache.solr.store.hdfs.HdfsLockFactory@281fce99); maxCacheMB=192.0 maxMergeSizeMB=16.0)) at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC]:0) at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:717) at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:709) Caused by: org.apache.lucene.index.CorruptIndexException: Problem reading index from NRTCachingDirectory(BlockDirectory(HdfsDirectory@hdfs://localhost:39347/solr_hdfs_home/recoverytest/core_node4/data/index lockFactory=org.apache.solr.store.hdfs.HdfsLockFactory@281fce99); maxCacheMB=192.0 maxMergeSizeMB=16.0) (resource=NRTCachingDirectory(BlockDirectory(HdfsDirectory@hdfs://localhost:39347/solr_hdfs_home/recoverytest/core_node4/data/index lockFactory=org.apache.solr.store.hdfs.HdfsLockFactory@281fce99); maxCacheMB=192.0 maxMergeSizeMB=16.0)) at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:160) at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:89) at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:179) at org.apache.lucene.index.ReadersAndUpdates.getReaderForMerge(ReadersAndUpdates.java:786) at org.apache.lucene.index.IndexWriter.lambda$mergeMiddle$19(IndexWriter.java:4906) at org.apache.lucene.index.MergePolicy$OneMerge.initMergeReaders(MergePolicy.java:418) at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4902) at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) Caused by: java.io.EOFException: position is negative at org.apache.hadoop.fs.FSInputStream.validatePositionedReadArgs(FSInputStream.java:103) at org.apache.hadoop.fs.FSInputStream.readFully(FSInputStream.java:118) at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:111) at org.apache.solr.store.hdfs.HdfsDirectory$HdfsIndexInput.readInternal(HdfsDirectory.java:251) at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:191) at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readByte(CustomBufferedIndexInput.java:45) at org.apache.lucene.store.DataInput.readVInt(DataInput.java:131) at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readVInt(CustomBufferedIndexInput.java:160) at org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader.<init>(VariableGapTermsIndexReader.java:73) at org.apache.lucene.codecs.blockterms.LuceneVarGapDocFreqInterval.fieldsProducer(LuceneVarGapDocFreqInterval.java:101) at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:327) at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:389) at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:117) ... 11 more FAILED: org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42869//control_collection: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42869//control_collection: Error opening new searcher at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1742) at org.apache.solr.cloud.SyncSliceTest.test(SyncSliceTest.java:92) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test Error Message: java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS Stack Trace: java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.junit.Assert.fail(Assert.java:89) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613) at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:98) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35793/uu/f/acollection0: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35793/uu/f/acollection0: Error opening new searcher at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1159) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480) at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test(HdfsWriteToMultipleCollectionsTest.java:128) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test Error Message: java.lang.AssertionError: expected:<0> but was:<14> Stack Trace: java.lang.AssertionError: expected:<0> but was:<14> at org.junit.Assert.fail(Assert.java:89) at org.junit.Assert.failNotEquals(Assert.java:835) at org.junit.Assert.assertEquals(Assert.java:647) at org.junit.Assert.assertEquals(Assert.java:633) at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:149) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33257/umw/rc/control_collection: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33257/umw/rc/control_collection: Error opening new searcher at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243) at org.apache.solr.BaseDistributedSearchTestCase.add(BaseDistributedSearchTestCase.java:573) at org.apache.solr.BaseDistributedSearchTestCase.indexDoc(BaseDistributedSearchTestCase.java:561) at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:214) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.StressHdfsTest.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39267/la/l/delete_data_dir: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39267/la/l/delete_data_dir: Error opening new searcher at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480) at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:190) at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41609/solr/recoverytest: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:41609/solr/recoverytest: Error opening new searcher at __randomizedtesting.SeedInfo.seed([4622B513F1888DDC:CE768AC95F74E024]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1159) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233) at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) Build Log: [...truncated 37225 lines...] 2> 1480650 ERROR (recoveryExecutor-6467-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.u.UpdateLog ERROR: didn't get to recover from tlog hdfs tlog{file=hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/tlog/tlog.0000000000000000005 refcount=2} 2> 1480650 WARN (recoveryExecutor-6467-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.u.UpdateLog Log replay finished. recoveryInfo=RecoveryInfo{adds=98 deletes=41 deleteByQuery=0 errors=1 positionOfStart=0} 2> 1480650 INFO (coreZkRegister-6446-thread-1-processing-n:127.0.0.1:35957_afa) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n8 2> 1480651 INFO (qtp368457112-12550) [n:127.0.0.1:41643_afa c:collection1 s:shard1 r:core_node12 x:collection1_shard1_replica_n10 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n10] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:40771/afa/collection1_shard1_replica_n15/&wt=javabin&version=2}{delete=[0-948 (-1709011828249460736)]} 0 0 2> 1480652 INFO (updateExecutor-6452-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.u.DefaultSolrCoreState Running recovery 2> 1480652 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true 2> 1480654 INFO (qtp909818695-10478) [n:127.0.0.1:40771_afa c:collection1 s:shard1 r:core_node17 x:collection1_shard1_replica_n15 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n15] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-948 (-1709011828249460736)]} 0 4 2> 1480657 INFO (qtp1052859918-10392) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-950 (-1709011828255752192)]} 0 0 2> 1480659 INFO (qtp29008378-10505) [n:127.0.0.1:45847_afa c:collection1 s:shard3 r:core_node14 x:collection1_shard3_replica_n13 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n13] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-950 (-1709011828257849344)]} 0 0 2> 1480660 INFO (qtp522140968-10467) [n:127.0.0.1:39029_afa c:collection1 s:shard3 r:core_node3 x:collection1_shard3_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n1] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-950 (-1709011828257849344)]} 0 0 2> 1480661 INFO (qtp1495782385-10574) [n:127.0.0.1:41645_afa c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n7 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n7] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-950 (-1709011828257849344)]} 0 3 2> 1480662 INFO (qtp1052859918-10708) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-951 (-1709011828262043648)]} 0 0 2> 1480665 INFO (qtp368457112-12548) [n:127.0.0.1:41643_afa c:collection1 s:shard1 r:core_node12 x:collection1_shard1_replica_n10 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n10] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:40771/afa/collection1_shard1_replica_n15/&wt=javabin&version=2}{delete=[0-951 (-1709011828263092224)]} 0 0 2> 1480666 INFO (qtp909818695-10482) [n:127.0.0.1:40771_afa c:collection1 s:shard1 r:core_node17 x:collection1_shard1_replica_n15 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n15] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-951 (-1709011828263092224)]} 0 2 2> 1480667 INFO (qtp1052859918-10395) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-952 (-1709011828266237952)]} 0 0 2> 1480669 INFO (qtp368457112-12547) [n:127.0.0.1:41643_afa c:collection1 s:shard1 r:core_node12 x:collection1_shard1_replica_n10 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n10] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:40771/afa/collection1_shard1_replica_n15/&wt=javabin&version=2}{delete=[0-952 (-1709011828268335104)]} 0 0 2> 1480670 INFO (qtp909818695-10480) [n:127.0.0.1:40771_afa c:collection1 s:shard1 r:core_node17 x:collection1_shard1_replica_n15 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n15] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-952 (-1709011828268335104)]} 0 1 2> 1480671 INFO (qtp1052859918-10713) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-953 (-1709011828270432256)]} 0 0 2> 1480681 INFO (qtp29008378-10508) [n:127.0.0.1:45847_afa c:collection1 s:shard3 r:core_node14 x:collection1_shard3_replica_n13 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n13] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-953 (-1709011828272529408)]} 0 0 2> 1480682 INFO (qtp522140968-10465) [n:127.0.0.1:39029_afa c:collection1 s:shard3 r:core_node3 x:collection1_shard3_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n1] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-953 (-1709011828272529408)]} 0 9 2> 1480684 INFO (qtp1495782385-10573) [n:127.0.0.1:41645_afa c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n7 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n7] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-953 (-1709011828272529408)]} 0 12 2> 1480686 INFO (qtp1052859918-10709) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-954 (-1709011828286160896)]} 0 1 2> 1480687 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy startupVersions size=100 range=[-1709011820735365120 to 1709011820302303235] 2> 1480688 INFO (qtp29008378-10504) [n:127.0.0.1:45847_afa c:collection1 s:shard3 r:core_node14 x:collection1_shard3_replica_n13 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n13] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-954 (-1709011828288258048)]} 0 0 2> 1480688 INFO (qtp1383425010-10492) [n:127.0.0.1:39859_afa c:collection1 s:shard2 r:core_node5 x:collection1_shard2_replica_n2 ] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/afa path=/admin/ping params={wt=javabin&version=2} hits=127 status=0 QTime=0 2> 1480689 INFO (qtp1383425010-10492) [n:127.0.0.1:39859_afa c:collection1 s:shard2 r:core_node5 x:collection1_shard2_replica_n2 ] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/afa path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0 2> 1480689 INFO (qtp522140968-10469) [n:127.0.0.1:39029_afa c:collection1 s:shard3 r:core_node3 x:collection1_shard3_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n1] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-954 (-1709011828288258048)]} 0 1 2> 1480690 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_n8] 2> 1480690 INFO (qtp1495782385-10572) [n:127.0.0.1:41645_afa c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n7 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n7] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-954 (-1709011828288258048)]} 0 3 2> 1480691 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.u.UpdateLog Starting to buffer updates. HDFSUpdateLog{state=ACTIVE, tlog=null} 2> 1480691 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_n8] as recovering, leader is [http://127.0.0.1:39859/afa/collection1_shard2_replica_n2/] and I am [http://127.0.0.1:35957/afa/collection1_shard2_replica_n8/] 2> 1480692 INFO (qtp1052859918-10393) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-957 (-1709011828292452352)]} 0 0 2> 1480693 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node18=2, core_node18_recovering=1, core_node11_recovering=3, core_node11=4, core_node5=4}, version=8} for startRecovering 2> 1480695 INFO (qtp29008378-10507) [n:127.0.0.1:45847_afa c:collection1 s:shard3 r:core_node14 x:collection1_shard3_replica_n13 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n13] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-957 (-1709011828295598080)]} 0 0 2> 1480696 INFO (qtp522140968-10468) [n:127.0.0.1:39029_afa c:collection1 s:shard3 r:core_node3 x:collection1_shard3_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n1] webapp=/afa path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:41645/afa/collection1_shard3_replica_n7/&wt=javabin&version=2}{delete=[0-957 (-1709011828295598080)]} 0 1 2> 1480697 INFO (qtp1495782385-10571) [n:127.0.0.1:41645_afa c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n7 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard3_replica_n7] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-957 (-1709011828295598080)]} 0 4 2> 1480698 INFO (qtp1052859918-10396) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{delete=[0-959 (-1709011828299792384)]} 0 1 2> 1480698 INFO (ChaosMonkey) [n:127.0.0.1:39859_afa ] o.a.s.c.ChaosMonkey monkey: only one active node in shard - monkey cannot kill :( 2> 1480700 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:39859/afa]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_n2&nodeName=127.0.0.1:35957_afa&coreNodeName=core_node11&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true] 2> 1480700 INFO (qtp1383425010-10494) [n:127.0.0.1:39859_afa c:collection1 s:shard2 r:core_node5 x:collection1_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node18=2, core_node18_recovering=1, core_node11_recovering=3, core_node11=4, core_node5=5}, version=9} for ensureTermsIsHigher 2> 1480700 INFO (qtp1383425010-10494) [n:127.0.0.1:39859_afa c:collection1 s:shard2 r:core_node5 x:collection1_shard2_replica_n2 ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2] webapp=/afa path=/update params={wt=javabin&version=2}{delete=[0-959 (-1709011828300840960)]} 0 1 2> 1480701 INFO (qtp1383425010-10491) [n:127.0.0.1:39859_afa x:collection1_shard2_replica_n2 ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node11, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true 2> 1480703 INFO (zkCallback-6457-thread-3) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveringCoreTermWatcher Start recovery on core_node11 because core's term is less than leader's term 2> 1480703 INFO (qtp1383425010-10491) [n:127.0.0.1:39859_afa ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_n2, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:35957_afa, coreNodeName=core_node11, onlyIfActiveCheckResult=false, nodeProps: core_node11:{ 2> "core":"collection1_shard2_replica_n8", 2> "core_node_name":"core_node11", 2> "dataDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/", 2> "node_name":"127.0.0.1:35957_afa", 2> "base_url":"http://127.0.0.1:35957/afa";, 2> "shared_storage":"true", 2> "collection":"collection1", 2> "state":"down", 2> "shard":"shard2", 2> "type":"NRT", 2> "force_set_state":"false", 2> "ulogDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/tlog"} 2> 1480704 WARN (updateExecutor-6452-thread-2-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1_shard2_replica_n8] coreNodeName=[core_node11] 2> 1480704 INFO (qtp1052859918-10711) [n:127.0.0.1:35529_afa c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp=/afa path=/update params={wt=javabin&version=2&CONTROL=TRUE}{add=[0-959 (1709011828302938112), 0-960 (1709011828302938113), 0-961 (1709011828303986688), 0-962 (1709011828303986689), 0-963 (1709011828303986690), 0-964 (1709011828303986691), 0-965 (1709011828303986692), 0-966 (1709011828303986693), 0-967 (1709011828303986694), 0-968 (1709011828303986695), ... (29 adds)]} 0 3 2> 1480705 INFO (qtp1383425010-10491) [n:127.0.0.1:39859_afa ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_n2, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:35957_afa, coreNodeName=core_node11, onlyIfActiveCheckResult=false, nodeProps: core_node11:{ 2> "core":"collection1_shard2_replica_n8", 2> "core_node_name":"core_node11", 2> "dataDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/", 2> "node_name":"127.0.0.1:35957_afa", 2> "base_url":"http://127.0.0.1:35957/afa";, 2> "shared_storage":"true", 2> "collection":"collection1", 2> "state":"down", 2> "shard":"shard2", 2> "type":"NRT", 2> "force_set_state":"false", 2> "ulogDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/tlog"} 2> 1480706 INFO (qtp1383425010-10491) [n:127.0.0.1:39859_afa ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_n2, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:35957_afa, coreNodeName=core_node11, onlyIfActiveCheckResult=false, nodeProps: core_node11:{ 2> "core":"collection1_shard2_replica_n8", 2> "core_node_name":"core_node11", 2> "dataDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/", 2> "node_name":"127.0.0.1:35957_afa", 2> "base_url":"http://127.0.0.1:35957/afa";, 2> "shared_storage":"true", 2> "collection":"collection1", 2> "state":"down", 2> "shard":"shard2", 2> "type":"NRT", 2> "force_set_state":"false", 2> "ulogDir":"hdfs://localhost:37975/solr_hdfs_home/collection1/core_node11/data/tlog"} 2> 1480709 ERROR (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Error while trying to recover. core=collection1_shard2_replica_n8:java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:39859/afa 2> at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122) 2> at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:191) 2> at org.apache.solr.cloud.RecoveryStrategy.sendPrepRecoveryCmd(RecoveryStrategy.java:874) 2> at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:611) 2> at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:330) 2> at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:309) 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) 2> at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) 2> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235) 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:39859/afa 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:710) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:312) 2> ... 5 more 2> Caused by: java.net.SocketException: Socket closed 2> at java.base/java.net.SocketInputStream.read(SocketInputStream.java:183) 2> at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) 2> at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) 2> at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) 2> at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) 2> at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) 2> at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) 2> at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:122) 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:580) 2> ... 6 more 2> 2> 1480710 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Recovery for core collection1_shard2_replica_n8 has been closed 2> 1480710 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x:collection1_shard2_replica_n8 c:collection1 s:shard2 r:core_node11) [n:127.0.0.1:35957_afa c:collection1 s:shard2 r:core_node11 x:collection1_shard2_replica_n8 ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false] 2> 1480710 INFO (recoveryExecutor-6454-thread-1-processing-n:127.0.0.1:35957_afa x: [...truncated too long message...] ts.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsSyncSliceTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=4622B513F1888DDC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0. Use '--warning-mode all' to show the individual deprecation warnings. See https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings BUILD SUCCESSFUL in 1h 58m 9s 91 actionable tasks: 91 executed Build step 'Invoke Gradle script' changed build result to SUCCESS Archiving artifacts java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:3069) at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2948) at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2929) at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3317) Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2 at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1800) at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:356) at hudson.remoting.Channel.call(Channel.java:1001) at hudson.FilePath.act(FilePath.java:1159) at hudson.FilePath.act(FilePath.java:1148) at hudson.FilePath.validateAntFileMask(FilePath.java:2927) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:752) at hudson.model.Build$BuildExecution.post2(Build.java:177) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:697) at hudson.model.Run.execute(Run.java:1931) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) Caused: hudson.FilePath$TunneledInterruptedException at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3319) at hudson.remoting.UserRequest.perform(UserRequest.java:211) at hudson.remoting.UserRequest.perform(UserRequest.java:54) at hudson.remoting.Request$2.run(Request.java:376) at hudson.remoting.InterceptingExecutorService.lambda$wrap$0(InterceptingExecutorService.java:78) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath.act(FilePath.java:1161) at hudson.FilePath.act(FilePath.java:1148) at hudson.FilePath.validateAntFileMask(FilePath.java:2927) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:752) at hudson.model.Build$BuildExecution.post2(Build.java:177) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:697) at hudson.model.Run.execute(Run.java:1931) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error? Recording test results [Checks API] No suitable checks publisher found. Build step 'Publish JUnit test result report' changed build result to UNSTABLE Email was triggered for: Unstable (Test Failures) Sending email for trigger: Unstable (Test Failures)
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
