Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/9/
20 tests failed. FAILED: org.apache.lucene.codecs.simpletext.TestSimpleTextLiveDocsFormat.testOverflow Error Message: Java heap space Stack Trace: java.lang.OutOfMemoryError: Java heap space at __randomizedtesting.SeedInfo.seed([84B2D01F5935B264:BF1AF8F05A043352]:0) at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78) at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51) at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164) at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150) at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141) at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126) at org.apache.lucene.codecs.simpletext.SimpleTextUtil.write(SimpleTextUtil.java:48) at org.apache.lucene.codecs.simpletext.SimpleTextLiveDocsFormat.writeLiveDocs(SimpleTextLiveDocsFormat.java:113) at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testSerialization(BaseLiveDocsFormatTestCase.java:129) at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testOverflow(BaseLiveDocsFormatTestCase.java:78) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) FAILED: org.apache.lucene.codecs.lucene50.TestLucene50LiveDocsFormat.testOverflow Error Message: Java heap space Stack Trace: java.lang.OutOfMemoryError: Java heap space at __randomizedtesting.SeedInfo.seed([ECD9E68EE67C6B0B:D771CE61E54DEA3D]:0) at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78) at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51) at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164) at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150) at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141) at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126) at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:71) at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213) at org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat.writeLiveDocs(Lucene50LiveDocsFormat.java:110) at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testSerialization(BaseLiveDocsFormatTestCase.java:129) at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testOverflow(BaseLiveDocsFormatTestCase.java:78) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) FAILED: org.apache.solr.cloud.RollingRestartTest.test Error Message: Unable to restart (#4): CloudJettyRunner [url=https://127.0.0.1:34865/sz/collection1_shard1_replica_n51] Stack Trace: java.lang.AssertionError: Unable to restart (#4): CloudJettyRunner [url=https://127.0.0.1:34865/sz/collection1_shard1_replica_n51] at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26:A31FB94B6B3011DE]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:103) at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest Error Message: ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:545) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393) at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331) at java.lang.Thread.run(Thread.java:748) Stack Trace: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:545) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393) at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest Error Message: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest: 1) Thread[id=1708, name=OverseerAutoScalingTriggerThread-73771535134621698-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128) at java.io.PrintStream.write(PrintStream.java:480) at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53) at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263) at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295) at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161) at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156) at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129) at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120) at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84) at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448) at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417) at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439) at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403) at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49) at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146) at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170) at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125) at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108) at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002) at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974) at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest: 1) Thread[id=1708, name=OverseerAutoScalingTriggerThread-73771535134621698-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128) at java.io.PrintStream.write(PrintStream.java:480) at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53) at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263) at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295) at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170) at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161) at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156) at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129) at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120) at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84) at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448) at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417) at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439) at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417) at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403) at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49) at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146) at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170) at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125) at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108) at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002) at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974) at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0) FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost Error Message: last state: DocCollection(testNodeLost//clusterstate.json/327)={ "replicationFactor":"10", "pullReplicas":"0", "router":{"name":"compositeId"}, "maxShardsPerNode":"5", "autoAddReplicas":"false", "nrtReplicas":"10", "tlogReplicas":"0", "autoCreated":"true", "shards":{ "shard20":{ "replicas":{ "core_node191":{ "core":"testNodeLost_shard20_replica_n191", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10093_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node234":{ "node_name":"127.0.0.1:10075_solr", "core":"testNodeLost_shard20_replica_n233", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node200":{ "core":"testNodeLost_shard20_replica_n200", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10106_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node199":{ "core":"testNodeLost_shard20_replica_n199", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10044_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node198":{ "core":"testNodeLost_shard20_replica_n198", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10062_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node197":{ "core":"testNodeLost_shard20_replica_n197", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10087_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node196":{ "core":"testNodeLost_shard20_replica_n196", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10052_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node195":{ "core":"testNodeLost_shard20_replica_n195", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10054_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node193":{ "core":"testNodeLost_shard20_replica_n193", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10058_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node192":{ "core":"testNodeLost_shard20_replica_n192", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10024_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"73330000-7fffffff", "state":"active"}, "shard10":{ "replicas":{ "core_node92":{ "core":"testNodeLost_shard10_replica_n92", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10024_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node91":{ "core":"testNodeLost_shard10_replica_n91", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10093_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node236":{ "node_name":"127.0.0.1:10050_solr", "core":"testNodeLost_shard10_replica_n235", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node93":{ "core":"testNodeLost_shard10_replica_n93", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10058_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node96":{ "core":"testNodeLost_shard10_replica_n96", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10052_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node95":{ "core":"testNodeLost_shard10_replica_n95", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10054_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node98":{ "core":"testNodeLost_shard10_replica_n98", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10062_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node100":{ "core":"testNodeLost_shard10_replica_n100", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10106_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node97":{ "core":"testNodeLost_shard10_replica_n97", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10087_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node99":{ "core":"testNodeLost_shard10_replica_n99", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10044_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"f3330000-ffffffff", "state":"active"}, "shard11":{ "replicas":{ "core_node108":{ "core":"testNodeLost_shard11_replica_n108", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10095_solr", "state":"down", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node107":{ "core":"testNodeLost_shard11_replica_n107", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10042_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node106":{ "core":"testNodeLost_shard11_replica_n106", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10099_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node105":{ "core":"testNodeLost_shard11_replica_n105", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10083_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node268":{ "node_name":"127.0.0.1:10000_solr", "core":"testNodeLost_shard11_replica_n267", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node102":{ "core":"testNodeLost_shard11_replica_n102", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10068_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node101":{ "core":"testNodeLost_shard11_replica_n101", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10114_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node110":{ "core":"testNodeLost_shard11_replica_n110", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10040_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node252":{ "node_name":"127.0.0.1:10115_solr", "core":"testNodeLost_shard11_replica_n251", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node109":{ "core":"testNodeLost_shard11_replica_n109", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10057_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"0-ccbffff", "state":"active"}, "shard12":{ "replicas":{ "core_node118":{ "core":"testNodeLost_shard12_replica_n118", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10048_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node228":{ "node_name":"127.0.0.1:10002_solr", "core":"testNodeLost_shard12_replica_n227", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node116":{ "core":"testNodeLost_shard12_replica_n116", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10032_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node115":{ "core":"testNodeLost_shard12_replica_n115", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10063_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node113":{ "core":"testNodeLost_shard12_replica_n113", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10027_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node112":{ "core":"testNodeLost_shard12_replica_n112", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10100_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node111":{ "core":"testNodeLost_shard12_replica_n111", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10097_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node220":{ "node_name":"127.0.0.1:10094_solr", "core":"testNodeLost_shard12_replica_n219", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node120":{ "core":"testNodeLost_shard12_replica_n120", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10075_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node260":{ "node_name":"127.0.0.1:10085_solr", "core":"testNodeLost_shard12_replica_n259", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}}, "range":"ccc0000-1998ffff", "state":"active"}, "shard2":{ "replicas":{ "core_node18":{ "core":"testNodeLost_shard2_replica_n18", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10048_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node218":{ "node_name":"127.0.0.1:10106_solr", "core":"testNodeLost_shard2_replica_n217", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node226":{ "node_name":"127.0.0.1:10097_solr", "core":"testNodeLost_shard2_replica_n225", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node258":{ "node_name":"127.0.0.1:10046_solr", "core":"testNodeLost_shard2_replica_n257", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node20":{ "core":"testNodeLost_shard2_replica_n20", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10075_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node12":{ "core":"testNodeLost_shard2_replica_n12", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10100_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node11":{ "core":"testNodeLost_shard2_replica_n11", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10097_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node13":{ "core":"testNodeLost_shard2_replica_n13", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10027_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node16":{ "core":"testNodeLost_shard2_replica_n16", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10032_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node15":{ "core":"testNodeLost_shard2_replica_n15", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10063_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"8ccc0000-9998ffff", "state":"active"}, "shard13":{ "replicas":{ "core_node129":{ "core":"testNodeLost_shard13_replica_n129", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10034_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node128":{ "core":"testNodeLost_shard13_replica_n128", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10043_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node127":{ "core":"testNodeLost_shard13_replica_n127", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10029_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node126":{ "core":"testNodeLost_shard13_replica_n126", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10108_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node246":{ "node_name":"127.0.0.1:10062_solr", "core":"testNodeLost_shard13_replica_n245", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node125":{ "core":"testNodeLost_shard13_replica_n125", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10059_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node124":{ "core":"testNodeLost_shard13_replica_n124", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10115_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node122":{ "core":"testNodeLost_shard13_replica_n122", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10124_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node254":{ "node_name":"127.0.0.1:10063_solr", "core":"testNodeLost_shard13_replica_n253", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node121":{ "core":"testNodeLost_shard13_replica_n121", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10113_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"19990000-2665ffff", "state":"active"}, "shard3":{ "replicas":{ "core_node29":{ "core":"testNodeLost_shard3_replica_n29", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10034_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node28":{ "core":"testNodeLost_shard3_replica_n28", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10043_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node248":{ "node_name":"127.0.0.1:10110_solr", "core":"testNodeLost_shard3_replica_n247", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node256":{ "node_name":"127.0.0.1:10068_solr", "core":"testNodeLost_shard3_replica_n255", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node21":{ "core":"testNodeLost_shard3_replica_n21", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10113_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node22":{ "core":"testNodeLost_shard3_replica_n22", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10124_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node25":{ "core":"testNodeLost_shard3_replica_n25", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10059_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node24":{ "core":"testNodeLost_shard3_replica_n24", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10115_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node27":{ "core":"testNodeLost_shard3_replica_n27", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10029_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node26":{ "core":"testNodeLost_shard3_replica_n26", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10108_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"99990000-a665ffff", "state":"active"}, "shard14":{ "replicas":{ "core_node139":{ "core":"testNodeLost_shard14_replica_n139", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10101_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node138":{ "core":"testNodeLost_shard14_replica_n138", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10109_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node137":{ "core":"testNodeLost_shard14_replica_n137", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10036_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node136":{ "core":"testNodeLost_shard14_replica_n136", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10045_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node135":{ "core":"testNodeLost_shard14_replica_n135", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10074_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node222":{ "node_name":"127.0.0.1:10034_solr", "core":"testNodeLost_shard14_replica_n221", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node133":{ "core":"testNodeLost_shard14_replica_n133", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10008_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node131":{ "core":"testNodeLost_shard14_replica_n131", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10056_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node262":{ "node_name":"127.0.0.1:10040_solr", "core":"testNodeLost_shard14_replica_n261", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node140":{ "core":"testNodeLost_shard14_replica_n140", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10116_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"26660000-3332ffff", "state":"active"}, "shard15":{ "replicas":{ "core_node206":{ "node_name":"127.0.0.1:10103_solr", "core":"testNodeLost_shard15_replica_n205", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node149":{ "core":"testNodeLost_shard15_replica_n149", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10050_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node148":{ "core":"testNodeLost_shard15_replica_n148", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10092_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node147":{ "core":"testNodeLost_shard15_replica_n147", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10110_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node146":{ "core":"testNodeLost_shard15_replica_n146", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10037_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node145":{ "core":"testNodeLost_shard15_replica_n145", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10125_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node144":{ "core":"testNodeLost_shard15_replica_n144", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10055_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node143":{ "core":"testNodeLost_shard15_replica_n143", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10025_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node142":{ "core":"testNodeLost_shard15_replica_n142", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10011_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node141":{ "core":"testNodeLost_shard15_replica_n141", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10084_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"33330000-3fffffff", "state":"active"}, "shard4":{ "replicas":{ "core_node39":{ "core":"testNodeLost_shard4_replica_n39", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10101_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node224":{ "node_name":"127.0.0.1:10031_solr", "core":"testNodeLost_shard4_replica_n223", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node40":{ "core":"testNodeLost_shard4_replica_n40", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10116_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node264":{ "node_name":"127.0.0.1:10011_solr", "core":"testNodeLost_shard4_replica_n263", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node31":{ "core":"testNodeLost_shard4_replica_n31", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10056_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node33":{ "core":"testNodeLost_shard4_replica_n33", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10008_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node36":{ "core":"testNodeLost_shard4_replica_n36", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10045_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node35":{ "core":"testNodeLost_shard4_replica_n35", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10074_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node38":{ "core":"testNodeLost_shard4_replica_n38", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10109_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node37":{ "core":"testNodeLost_shard4_replica_n37", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10036_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"a6660000-b332ffff", "state":"active"}, "shard5":{ "replicas":{ "core_node41":{ "core":"testNodeLost_shard5_replica_n41", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10084_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node43":{ "core":"testNodeLost_shard5_replica_n43", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10025_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node42":{ "core":"testNodeLost_shard5_replica_n42", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10011_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node45":{ "core":"testNodeLost_shard5_replica_n45", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10125_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node44":{ "core":"testNodeLost_shard5_replica_n44", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10055_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node47":{ "core":"testNodeLost_shard5_replica_n47", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10110_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node46":{ "core":"testNodeLost_shard5_replica_n46", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10037_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node49":{ "core":"testNodeLost_shard5_replica_n49", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10050_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node48":{ "core":"testNodeLost_shard5_replica_n48", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10092_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node208":{ "node_name":"127.0.0.1:10059_solr", "core":"testNodeLost_shard5_replica_n207", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}}, "range":"b3330000-bfffffff", "state":"active"}, "shard16":{ "replicas":{ "core_node159":{ "core":"testNodeLost_shard16_replica_n159", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10102_solr", "state":"down", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node157":{ "core":"testNodeLost_shard16_replica_n157", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10094_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node156":{ "core":"testNodeLost_shard16_replica_n156", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10090_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node155":{ "core":"testNodeLost_shard16_replica_n155", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10039_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node154":{ "core":"testNodeLost_shard16_replica_n154", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10103_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node153":{ "core":"testNodeLost_shard16_replica_n153", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10010_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node230":{ "node_name":"127.0.0.1:10078_solr", "core":"testNodeLost_shard16_replica_n229", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node152":{ "core":"testNodeLost_shard16_replica_n152", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10089_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node151":{ "core":"testNodeLost_shard16_replica_n151", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10122_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node160":{ "core":"testNodeLost_shard16_replica_n160", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10007_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"40000000-4ccbffff", "state":"active"}, "shard17":{ "replicas":{ "core_node169":{ "core":"testNodeLost_shard17_replica_n169", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10016_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node168":{ "core":"testNodeLost_shard17_replica_n168", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10009_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node166":{ "core":"testNodeLost_shard17_replica_n166", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10076_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node210":{ "node_name":"127.0.0.1:10053_solr", "core":"testNodeLost_shard17_replica_n209", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node165":{ "core":"testNodeLost_shard17_replica_n165", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10071_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node164":{ "core":"testNodeLost_shard17_replica_n164", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10053_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node163":{ "core":"testNodeLost_shard17_replica_n163", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10085_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node161":{ "core":"testNodeLost_shard17_replica_n161", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10002_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node270":{ "node_name":"127.0.0.1:10009_solr", "core":"testNodeLost_shard17_replica_n269", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node170":{ "core":"testNodeLost_shard17_replica_n170", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10079_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"4ccc0000-5998ffff", "state":"active"}, "shard18":{ "replicas":{ "core_node180":{ "core":"testNodeLost_shard18_replica_n180", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10112_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node202":{ "node_name":"127.0.0.1:10072_solr", "core":"testNodeLost_shard18_replica_n201", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node179":{ "core":"testNodeLost_shard18_replica_n179", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10046_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node178":{ "core":"testNodeLost_shard18_replica_n178", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10000_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node177":{ "core":"testNodeLost_shard18_replica_n177", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10123_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node176":{ "core":"testNodeLost_shard18_replica_n176", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10077_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node175":{ "core":"testNodeLost_shard18_replica_n175", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10069_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node174":{ "core":"testNodeLost_shard18_replica_n174", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10031_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node172":{ "core":"testNodeLost_shard18_replica_n172", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10006_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node171":{ "core":"testNodeLost_shard18_replica_n171", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10086_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"59990000-6665ffff", "state":"active"}, "shard19":{ "replicas":{ "core_node190":{ "core":"testNodeLost_shard19_replica_n190", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10030_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node238":{ "node_name":"127.0.0.1:10122_solr", "core":"testNodeLost_shard19_replica_n237", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node214":{ "node_name":"127.0.0.1:10037_solr", "core":"testNodeLost_shard19_replica_n213", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node189":{ "core":"testNodeLost_shard19_replica_n189", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10013_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node188":{ "core":"testNodeLost_shard19_replica_n188", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10072_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node242":{ "node_name":"127.0.0.1:10116_solr", "core":"testNodeLost_shard19_replica_n241", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node185":{ "core":"testNodeLost_shard19_replica_n185", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10078_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node184":{ "core":"testNodeLost_shard19_replica_n184", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10028_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node182":{ "core":"testNodeLost_shard19_replica_n182", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10111_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node181":{ "core":"testNodeLost_shard19_replica_n181", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10001_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"66660000-7332ffff", "state":"active"}, "shard1":{ "replicas":{ "core_node1":{ "core":"testNodeLost_shard1_replica_n1", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10114_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node266":{ "node_name":"127.0.0.1:10030_solr", "core":"testNodeLost_shard1_replica_n265", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node6":{ "core":"testNodeLost_shard1_replica_n6", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10099_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node10":{ "core":"testNodeLost_shard1_replica_n10", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10040_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node7":{ "core":"testNodeLost_shard1_replica_n7", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10042_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node8":{ "core":"testNodeLost_shard1_replica_n8", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10095_solr", "state":"down", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node9":{ "core":"testNodeLost_shard1_replica_n9", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10057_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node2":{ "core":"testNodeLost_shard1_replica_n2", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10068_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node250":{ "node_name":"127.0.0.1:10069_solr", "core":"testNodeLost_shard1_replica_n249", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node5":{ "core":"testNodeLost_shard1_replica_n5", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10083_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"80000000-8ccbffff", "state":"active"}, "shard6":{ "replicas":{ "core_node60":{ "core":"testNodeLost_shard6_replica_n60", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10007_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node52":{ "core":"testNodeLost_shard6_replica_n52", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10089_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node51":{ "core":"testNodeLost_shard6_replica_n51", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10122_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node232":{ "node_name":"127.0.0.1:10125_solr", "core":"testNodeLost_shard6_replica_n231", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node54":{ "core":"testNodeLost_shard6_replica_n54", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10103_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node53":{ "core":"testNodeLost_shard6_replica_n53", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10010_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node56":{ "core":"testNodeLost_shard6_replica_n56", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10090_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node55":{ "core":"testNodeLost_shard6_replica_n55", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10039_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node57":{ "core":"testNodeLost_shard6_replica_n57", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10094_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node59":{ "core":"testNodeLost_shard6_replica_n59", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10102_solr", "state":"down", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"c0000000-cccbffff", "state":"active"}, "shard7":{ "replicas":{ "core_node70":{ "core":"testNodeLost_shard7_replica_n70", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10079_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node61":{ "core":"testNodeLost_shard7_replica_n61", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10002_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node63":{ "core":"testNodeLost_shard7_replica_n63", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10085_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node212":{ "node_name":"127.0.0.1:10109_solr", "core":"testNodeLost_shard7_replica_n211", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node65":{ "core":"testNodeLost_shard7_replica_n65", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10071_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node64":{ "core":"testNodeLost_shard7_replica_n64", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10053_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node66":{ "core":"testNodeLost_shard7_replica_n66", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10076_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node69":{ "core":"testNodeLost_shard7_replica_n69", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10016_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node272":{ "node_name":"127.0.0.1:10087_solr", "core":"testNodeLost_shard7_replica_n271", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node68":{ "core":"testNodeLost_shard7_replica_n68", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10009_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"cccc0000-d998ffff", "state":"active"}, "shard8":{ "replicas":{ "core_node80":{ "core":"testNodeLost_shard8_replica_n80", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10112_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node204":{ "node_name":"127.0.0.1:10056_solr", "core":"testNodeLost_shard8_replica_n203", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node72":{ "core":"testNodeLost_shard8_replica_n72", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10006_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node71":{ "core":"testNodeLost_shard8_replica_n71", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10086_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node74":{ "core":"testNodeLost_shard8_replica_n74", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10031_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node76":{ "core":"testNodeLost_shard8_replica_n76", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10077_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node75":{ "core":"testNodeLost_shard8_replica_n75", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10069_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node78":{ "core":"testNodeLost_shard8_replica_n78", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10000_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node77":{ "core":"testNodeLost_shard8_replica_n77", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10123_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node79":{ "core":"testNodeLost_shard8_replica_n79", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10046_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"d9990000-e665ffff", "state":"active"}, "shard9":{ "replicas":{ "core_node90":{ "core":"testNodeLost_shard9_replica_n90", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10030_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node81":{ "core":"testNodeLost_shard9_replica_n81", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10001_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node216":{ "node_name":"127.0.0.1:10100_solr", "core":"testNodeLost_shard9_replica_n215", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node82":{ "core":"testNodeLost_shard9_replica_n82", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10111_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node85":{ "core":"testNodeLost_shard9_replica_n85", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10078_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node84":{ "core":"testNodeLost_shard9_replica_n84", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10028_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node244":{ "node_name":"127.0.0.1:10008_solr", "core":"testNodeLost_shard9_replica_n243", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node89":{ "core":"testNodeLost_shard9_replica_n89", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10013_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node240":{ "node_name":"127.0.0.1:10024_solr", "core":"testNodeLost_shard9_replica_n239", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node88":{ "core":"testNodeLost_shard9_replica_n88", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10072_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}}, "range":"e6660000-f332ffff", "state":"active"}}} Stack Trace: java.util.concurrent.TimeoutException: last state: DocCollection(testNodeLost//clusterstate.json/327)={ "replicationFactor":"10", "pullReplicas":"0", "router":{"name":"compositeId"}, "maxShardsPerNode":"5", "autoAddReplicas":"false", "nrtReplicas":"10", "tlogReplicas":"0", "autoCreated":"true", "shards":{ "shard20":{ "replicas":{ "core_node191":{ "core":"testNodeLost_shard20_replica_n191", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10093_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node234":{ "node_name":"127.0.0.1:10075_solr", "core":"testNodeLost_shard20_replica_n233", "state":"active", "INDEX.sizeInBytes":1, "type":"NRT"}, "core_node200":{ "core":"testNodeLost_shard20_replica_n200", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10106_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node199":{ "core":"testNodeLost_shard20_replica_n199", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10044_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node198":{ "core":"testNodeLost_shard20_replica_n198", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10062_solr", "state":"active", "type":"NRT", "SEARCHER.searcher.numDocs":0}, "core_node197":{ "core":"testNodeLost_shard20_replica_n197", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":1, "node_name":"127.0.0.1:10087_solr", "state":"active", "type":"NRT", [...truncated too long message...] [junit4] > 6) Thread[id=95737, name=searcherExecutor-14160-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 7) Thread[id=95771, name=searcherExecutor-14171-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=95584, name=searcherExecutor-14121-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 2) Thread[id=95703, name=searcherExecutor-14149-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 3) Thread[id=95548, name=searcherExecutor-14110-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 4) Thread[id=95506, name=searcherExecutor-14099-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 5) Thread[id=95626, name=searcherExecutor-14132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 6) Thread[id=95737, name=searcherExecutor-14160-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 7) Thread[id=95771, name=searcherExecutor-14171-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0) [junit4] Completed [618/805 (7!)] on J2 in 461.37s, 1 test, 1 failure, 2 errors <<< FAILURES! [...truncated 557 lines...] [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20180509_211453_82688593682244439369.sysout [junit4] >>> JVM J0 emitted unexpected output (verbatim) ---- [junit4] java.lang.OutOfMemoryError: Java heap space [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/heapdumps/java_pid14928.hprof ... [junit4] Heap dump file created [467286832 bytes in 3.935 secs] [junit4] <<< JVM J0: EOF ---- [...truncated 9437 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:661: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created: * java_pid14928.hprof * java_pid3672.hprof * java_pid9480.hprof Total time: 385 minutes 35 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org