Build: https://ci-builds.apache.org/job/Solr/job/Solr-NightlyTests-main/169/
6 tests failed. FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test Error Message: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37977/j_gao/a/onenodecollection: Error opening new searcher Stack Trace: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37977/j_gao/a/onenodecollection: Error opening new searcher at __randomizedtesting.SeedInfo.seed([E62287A7DF2183FC:6E76B87D71DDEE04]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480) at org.apache.solr.cloud.BasicDistributedZk2Test.addAndQueryDocs(BasicDistributedZk2Test.java:201) at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:167) at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:76) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test Error Message: java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS Stack Trace: java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS at org.junit.Assert.fail(Assert.java:89) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613) at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:98) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test Error Message: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request Stack Trace: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request at __randomizedtesting.SeedInfo.seed([E62287A7DF2183FC:6E76B87D71DDEE04]:0) at org.apache.solr.client.solrj.impl.LBSolrClient$ServerIterator.nextOrError(LBSolrClient.java:221) at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:356) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1159) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1004) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1019) at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test(HdfsWriteToMultipleCollectionsTest.java:129) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33143/acollection0: Error from server at null: parsing error at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) ... 55 more FAILED: org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod Error Message: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest: 1) Thread[id=194297, name=aliveCheckExecutor-78590-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at [email protected]/java.lang.Thread.run(Thread.java:834) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest: 1) Thread[id=194297, name=aliveCheckExecutor-78590-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at [email protected]/java.lang.Thread.run(Thread.java:834) at __randomizedtesting.SeedInfo.seed([E62287A7DF2183FC]:0) FAILED: org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod Error Message: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=194297, name=aliveCheckExecutor-78590-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at [email protected]/java.lang.Thread.run(Thread.java:834) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=194297, name=aliveCheckExecutor-78590-thread-1, state=TIMED_WAITING, group=TGRP-HdfsWriteToMultipleCollectionsTest] at [email protected]/jdk.internal.misc.Unsafe.park(Native Method) at [email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at [email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at [email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at [email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at [email protected]/java.lang.Thread.run(Thread.java:834) at __randomizedtesting.SeedInfo.seed([E62287A7DF2183FC]:0) FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test Error Message: java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals. Stack Trace: java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals. at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613) at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:236) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370) at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826) at java.base/java.lang.Thread.run(Thread.java:834) Build Log: [...truncated 713158 lines...] 2> 5334989 ERROR (recoveryExecutor-78337-thread-1-processing-n:127.0.0.1:37093_mfzzo%2Fv x:collection1_shard3_replica_n6 c:collection1 s:shard3 r:core_node9) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.u.UpdateLog ERROR: didn't get to recover from tlog hdfs tlog{file=hdfs://localhost.localdomain:39345/solr_hdfs_home/collection1/core_node9/data/tlog/tlog.0000000000000000010 refcount=2} 2> 5334989 WARN (recoveryExecutor-78337-thread-1-processing-n:127.0.0.1:37093_mfzzo%2Fv x:collection1_shard3_replica_n6 c:collection1 s:shard3 r:core_node9) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.u.UpdateLog Log replay finished. recoveryInfo=RecoveryInfo{adds=310 deletes=136 deleteByQuery=1 errors=1 positionOfStart=0} 2> 5334989 INFO (coreZkRegister-78317-thread-1-processing-n:127.0.0.1:37093_mfzzo%2Fv) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.c.ZkController I am the leader, no recovery necessary 2> 5334990 INFO (qtp1459592007-182300) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@34b5c9e7 2> 5334990 WARN (qtp1459592007-182300) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.c.CoreContainer Could not roll index writer after tragedy 2> 5334991 ERROR (qtp1459592007-182300) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Server error writing document id 0-429 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.server.Server.handle(Server.java:516) 2> at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) 2> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) 2> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:907) 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:920) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1527) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1519) 2> at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:958) 2> at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:337) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) 2> ... 63 more 2> Caused by: java.io.EOFException: read past EOF: _1.fdt 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:185) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:93) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:67) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.doDecompress(DeflateWithPresetDictCompressionMode.java:85) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.decompress(DeflateWithPresetDictCompressionMode.java:143) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.doReset(Lucene90CompressingStoredFieldsReader.java:500) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.reset(Lucene90CompressingStoredFieldsReader.java:424) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.document(Lucene90CompressingStoredFieldsReader.java:607) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.copyOneDoc(Lucene90CompressingStoredFieldsWriter.java:521) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.merge(Lucene90CompressingStoredFieldsWriter.java:643) 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:220) 2> at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:257) 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:109) 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4967) 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) 2> at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) 2> at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) 2> 2> 5334991 ERROR (qtp1459592007-182300) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.s.HttpSolrCall org.apache.solr.common.SolrException: Server error writing document id 0-429 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.server.Server.handle(Server.java:516) 2> at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) 2> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) 2> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:907) 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:920) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1527) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1519) 2> at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:958) 2> at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:337) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) 2> ... 63 more 2> Caused by: java.io.EOFException: read past EOF: _1.fdt 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:185) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:93) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:67) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.doDecompress(DeflateWithPresetDictCompressionMode.java:85) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.decompress(DeflateWithPresetDictCompressionMode.java:143) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.doReset(Lucene90CompressingStoredFieldsReader.java:500) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.reset(Lucene90CompressingStoredFieldsReader.java:424) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.document(Lucene90CompressingStoredFieldsReader.java:607) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.copyOneDoc(Lucene90CompressingStoredFieldsWriter.java:521) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.merge(Lucene90CompressingStoredFieldsWriter.java:643) 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:220) 2> at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:257) 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:109) 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4967) 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) 2> at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) 2> at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) 2> 2> 5334992 INFO (qtp1459592007-182304) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2e35eb12 2> 5334993 INFO (StoppableIndexingThread) [ ] o.a.s.c.s.i.BaseCloudSolrClient request was not communication error it seems 2> 5334993 INFO (StoppableIndexingThread) [ ] o.a.s.c.s.i.BaseCloudSolrClient Request to collection [collection1] failed due to (500) org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 0-429 to the index, retry=0 maxRetries=5 commError=false errorCode=500 2> REQUEST FAILED for id=0-429 2> org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 0-429 to the index 2> at org.apache.solr.client.solrj.impl.CloudSolrClient.getRouteException(CloudSolrClient.java:124) 2> at org.apache.solr.client.solrj.impl.CloudSolrClient.getRouteException(CloudSolrClient.java:46) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:577) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1064) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) 2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) 2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243) 2> at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:181) 2> at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2> Caused by: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 0-429 to the index 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) 2> at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) 2> at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.lambda$directUpdate$0(BaseCloudSolrClient.java:553) 2> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235) 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> 5334993 WARN (qtp1459592007-182304) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.c.CoreContainer Could not roll index writer after tragedy 2> 5334993 INFO (qtp1459592007-182301) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@42182f8f 2> 5334993 WARN (qtp1459592007-182301) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.c.CoreContainer Could not roll index writer after tragedy 2> 5334994 ERROR (qtp1459592007-182301) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Server error writing document id 1-482 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.server.Server.handle(Server.java:516) 2> at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) 2> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) 2> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:907) 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:920) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1527) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1519) 2> at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:958) 2> at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:337) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) 2> ... 63 more 2> Caused by: java.io.EOFException: read past EOF: _1.fdt 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:185) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:93) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:67) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.doDecompress(DeflateWithPresetDictCompressionMode.java:85) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.decompress(DeflateWithPresetDictCompressionMode.java:143) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.doReset(Lucene90CompressingStoredFieldsReader.java:500) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.reset(Lucene90CompressingStoredFieldsReader.java:424) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.document(Lucene90CompressingStoredFieldsReader.java:607) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.copyOneDoc(Lucene90CompressingStoredFieldsWriter.java:521) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.merge(Lucene90CompressingStoredFieldsWriter.java:643) 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:220) 2> at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:257) 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:109) 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4967) 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) 2> at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) 2> at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) 2> 2> 5334994 ERROR (qtp1459592007-182301) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.s.HttpSolrCall org.apache.solr.common.SolrException: Server error writing document id 1-482 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.server.Server.handle(Server.java:516) 2> at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) 2> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) 2> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:907) 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:920) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1527) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1519) 2> at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:958) 2> at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:337) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) 2> ... 63 more 2> Caused by: java.io.EOFException: read past EOF: _1.fdt 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:185) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:93) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:67) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.doDecompress(DeflateWithPresetDictCompressionMode.java:85) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.decompress(DeflateWithPresetDictCompressionMode.java:143) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.doReset(Lucene90CompressingStoredFieldsReader.java:500) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.reset(Lucene90CompressingStoredFieldsReader.java:424) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.document(Lucene90CompressingStoredFieldsReader.java:607) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.copyOneDoc(Lucene90CompressingStoredFieldsWriter.java:521) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.merge(Lucene90CompressingStoredFieldsWriter.java:643) 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:220) 2> at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:257) 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:109) 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4967) 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) 2> at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) 2> at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) 2> 2> 5334995 INFO (StoppableIndexingThread) [ ] o.a.s.c.s.i.BaseCloudSolrClient request was not communication error it seems 2> 5334995 INFO (StoppableIndexingThread) [ ] o.a.s.c.s.i.BaseCloudSolrClient Request to collection [collection1] failed due to (500) org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 1-482 to the index, retry=0 maxRetries=5 commError=false errorCode=500 2> REQUEST FAILED for id=1-482 2> org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 1-482 to the index 2> at org.apache.solr.client.solrj.impl.CloudSolrClient.getRouteException(CloudSolrClient.java:124) 2> at org.apache.solr.client.solrj.impl.CloudSolrClient.getRouteException(CloudSolrClient.java:46) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:577) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1064) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:925) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:859) 2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226) 2> at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243) 2> at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:181) 2> at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2> Caused by: org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37093/mfzzo/v/collection1_shard3_replica_n6: Server error writing document id 1-482 to the index 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:696) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268) 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) 2> at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389) 2> at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359) 2> at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.lambda$directUpdate$0(BaseCloudSolrClient.java:553) 2> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235) 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> 5335008 ERROR (qtp1459592007-182304) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Server error writing document id 2-426 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713) 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) 2> at org.eclipse.jetty.server.Server.handle(Server.java:516) 2> at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) 2> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) 2> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) 2> at java.base/java.lang.Thread.run(Thread.java:834) 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:907) 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:920) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1527) 2> at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1519) 2> at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:958) 2> at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:337) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) 2> ... 63 more 2> Caused by: java.io.EOFException: read past EOF: _1.fdt 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.refill(CustomBufferedIndexInput.java:185) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:93) 2> at org.apache.solr.store.blockcache.CustomBufferedIndexInput.readBytes(CustomBufferedIndexInput.java:67) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.doDecompress(DeflateWithPresetDictCompressionMode.java:85) 2> at org.apache.lucene.codecs.lucene90.DeflateWithPresetDictCompressionMode$DeflateWithPresetDictDecompressor.decompress(DeflateWithPresetDictCompressionMode.java:143) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.doReset(Lucene90CompressingStoredFieldsReader.java:500) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader$BlockState.reset(Lucene90CompressingStoredFieldsReader.java:424) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsReader.document(Lucene90CompressingStoredFieldsReader.java:607) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.copyOneDoc(Lucene90CompressingStoredFieldsWriter.java:521) 2> at org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter.merge(Lucene90CompressingStoredFieldsWriter.java:643) 2> at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:220) 2> at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:257) 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:109) 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4967) 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4503) 2> at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201) 2> at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6255) 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:636) 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:697) 2> 2> 5335009 ERROR (qtp1459592007-182304) [n:127.0.0.1:37093_mfzzo%2Fv c:collection1 s:shard3 r:core_node9 x:collection1_shard3_replica_n6 ] o.a.s.s.HttpSolrCall org.apache.solr.common.SolrException: Server error writing document id 2-426 to the index 2> at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241) 2> at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:79) 2> at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:263) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:502) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:343) 2> at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:229) 2> at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:249) 2> at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:106) 2> at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:339) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:284) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:338) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:239) 2> at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:303) 2> at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:283) 2> at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:196) 2> at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:128) 2> at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:122) 2> at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) 2> at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) 2> at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82) 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212) 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2617) 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:829) 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:569) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:506) 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:433) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:184) 2> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) 2> at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350) 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 2> at org [...truncated too long message...] ting:compileTestJava > Task :solr:contrib:scripting:copyTestResources NO-SOURCE > Task :solr:contrib:scripting:processTestResources > Task :solr:contrib:scripting:testClasses > Task :solr:contrib:scripting:test :solr:contrib:scripting:test (SUCCESS): 22 test(s), 1 skipped > Task :solr:contrib:scripting:wipeTaskTemp The slowest tests (exceeding 500 ms) during this run: 3212.17s TestStressLiveNodes.testStress (:solr:core) 742.61s RollingRestartTest.test (:solr:core) 667.55s HdfsTlogReplayBufferedWhileIndexingTest.test (:solr:core) 444.26s TestDistributedSearch.test (:solr:core) 346.28s HdfsBasicDistributedZkTest.test (:solr:core) 265.83s TestRangeQuery.testRandomRangeQueries (:solr:core) 239.68s TestIntervalFaceting.testRandom (:solr:core) 208.93s TestRangeQuery.testCompareTypesRandomRangeQueries (:solr:core) 202.64s TestRandomCollapseQParserPlugin.testRandomCollpaseWithSort (:solr:core) 184.86s TestDistributedStatsComponentCardinality.test (:solr:core) The slowest suites (exceeding 1s) during this run: 3213.91s TestStressLiveNodes (:solr:core) 743.19s RollingRestartTest (:solr:core) 671.81s HdfsTlogReplayBufferedWhileIndexingTest (:solr:core) 477.76s TestRangeQuery (:solr:core) 444.65s TestDistributedSearch (:solr:core) 349.16s HdfsBasicDistributedZkTest (:solr:core) 284.13s TestReplicationHandler (:solr:core) 271.15s TestJsonFacets (:solr:core) 241.65s TestIntervalFaceting (:solr:core) 221.39s TestRandomCollapseQParserPlugin (:solr:core) ERROR: The following test(s) have failed: - org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt - org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod (:solr:core) Test output: /home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.txt Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.classMethod" -Ptests.jvms=4 -Ptests.haltonfailure=false -Ptests.jvmargs=-XX:TieredStopAtLevel=1 -Ptests.seed=E62287A7DF2183FC -Ptests.multiplier=2 -Ptests.nightly=true -Ptests.badapples=false -Ptests.file.encoding=US-ASCII -Ptests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Solr/Solr-NightlyTests-main/test-data/enwiki.random.lines.txt Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0. Use '--warning-mode all' to show the individual deprecation warnings. See https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings BUILD SUCCESSFUL in 1h 43m 50s 91 actionable tasks: 91 executed Build step 'Invoke Gradle script' changed build result to SUCCESS Archiving artifacts java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:3069) at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2948) at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2929) at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3317) Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene1 at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1800) at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:356) at hudson.remoting.Channel.call(Channel.java:1001) at hudson.FilePath.act(FilePath.java:1159) at hudson.FilePath.act(FilePath.java:1148) at hudson.FilePath.validateAntFileMask(FilePath.java:2927) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:752) at hudson.model.Build$BuildExecution.post2(Build.java:177) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:697) at hudson.model.Run.execute(Run.java:1931) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) Caused: hudson.FilePath$TunneledInterruptedException at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3319) at hudson.remoting.UserRequest.perform(UserRequest.java:211) at hudson.remoting.UserRequest.perform(UserRequest.java:54) at hudson.remoting.Request$2.run(Request.java:376) at hudson.remoting.InterceptingExecutorService.lambda$wrap$0(InterceptingExecutorService.java:78) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath.act(FilePath.java:1161) at hudson.FilePath.act(FilePath.java:1148) at hudson.FilePath.validateAntFileMask(FilePath.java:2927) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:803) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:752) at hudson.model.Build$BuildExecution.post2(Build.java:177) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:697) at hudson.model.Run.execute(Run.java:1931) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error? Recording test results [Checks API] No suitable checks publisher found. Build step 'Publish JUnit test result report' changed build result to UNSTABLE Email was triggered for: Unstable (Test Failures) Sending email for trigger: Unstable (Test Failures)
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
