Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/271/

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderTragicEventTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [TransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.TransactionLog  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)  at 
org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:467)  at 
org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1323)  at 
org.apache.solr.update.UpdateLog.add(UpdateLog.java:571)  at 
org.apache.solr.update.UpdateLog.add(UpdateLog.java:551)  at 
org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:345)
  at 
org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:283)
  at 
org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:233)
  at 
org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
  at 
org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
  at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:951)
  at 
org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1167)
  at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:634)
  at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
  at 
org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)  
at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
  at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311) 
 at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)  at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
  at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276) 
 at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)  at 
org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)  at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
  at 
org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:109)
  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)  
at 
org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
  at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
  at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2541)  at 
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)  at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) 
 at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
  at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:674)  
at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) 
 at org.eclipse.jetty.server.Server.handle(Server.java:531)  at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)  at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:291)  at 
org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:151)  at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)  at 
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
  at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
  at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
  at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
  at 
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) 
 at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [TransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.TransactionLog
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188)
        at 
org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:467)
        at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1323)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:571)
        at org.apache.solr.update.UpdateLog.add(UpdateLog.java:551)
        at 
org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:345)
        at 
org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:283)
        at 
org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:233)
        at 
org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:951)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1167)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:634)
        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
        at 
org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
        at 
org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311)
        at 
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
        at 
org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276)
        at 
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
        at 
org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
        at 
org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:109)
        at 
org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
        at 
org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2541)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:674)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
        at org.eclipse.jetty.server.Server.handle(Server.java:531)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:291)
        at 
org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:151)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
        at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
        at 
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([14F869F052BC897B]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:304)
        at sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ShardRoutingTest.test

Error Message:
expected:<3> but was:<4>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<4>
        at 
__randomizedtesting.SeedInfo.seed([14F869F052BC897B:9CAC562AFC40E483]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.ShardRoutingTest.doTestNumRequests(ShardRoutingTest.java:279)
        at 
org.apache.solr.cloud.ShardRoutingTest.test(ShardRoutingTest.java:110)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
There are still nodes recoverying - waited for 60 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 60 
seconds
        at 
__randomizedtesting.SeedInfo.seed([14F869F052BC897B:9CAC562AFC40E483]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:921)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1478)
        at 
org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:397)
        at 
org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:108)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1008)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:983)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.document.TestLatLonShapeQueries.testRandomBig

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
        at 
__randomizedtesting.SeedInfo.seed([3643D60F496A3CC6:B114AB80D8334046]:0)
        at java.math.BigDecimal.valueOf(BigDecimal.java:1221)
        at java.math.BigDecimal.valueOf(BigDecimal.java:1198)
        at java.math.MutableBigInteger.toBigDecimal(MutableBigInteger.java:214)
        at java.math.BigDecimal.divideAndRound(BigDecimal.java:4333)
        at java.math.BigDecimal.setScale(BigDecimal.java:2470)
        at java.math.BigDecimal.toBigInteger(BigDecimal.java:3025)
        at org.apache.lucene.util.TestUtil.nextLong(TestUtil.java:443)
        at 
org.apache.lucene.geo.GeoTestUtil.nextDoubleInternal(GeoTestUtil.java:71)
        at org.apache.lucene.geo.GeoTestUtil.nextLongitude(GeoTestUtil.java:40)
        at 
org.apache.lucene.geo.GeoTestUtil.nextBoxInternal(GeoTestUtil.java:423)
        at org.apache.lucene.geo.GeoTestUtil.nextPolygon(GeoTestUtil.java:405)
        at 
org.apache.lucene.document.TestLatLonShapeQueries.doTestRandom(TestLatLonShapeQueries.java:126)
        at 
org.apache.lucene.document.TestLatLonShapeQueries.testRandomBig(TestLatLonShapeQueries.java:106)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)




Build Log:
[...truncated 10319 lines...]
   [junit4] Suite: org.apache.lucene.document.TestLatLonShapeQueries
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestLatLonShapeQueries -Dtests.method=testRandomBig 
-Dtests.seed=3643D60F496A3CC6 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=bg -Dtests.timezone=PRC -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII
   [junit4] ERROR    234s J0 | TestLatLonShapeQueries.testRandomBig <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit 
exceeded
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([3643D60F496A3CC6:B114AB80D8334046]:0)
   [junit4]    >        at java.math.BigDecimal.valueOf(BigDecimal.java:1221)
   [junit4]    >        at java.math.BigDecimal.valueOf(BigDecimal.java:1198)
   [junit4]    >        at 
java.math.MutableBigInteger.toBigDecimal(MutableBigInteger.java:214)
   [junit4]    >        at 
java.math.BigDecimal.divideAndRound(BigDecimal.java:4333)
   [junit4]    >        at java.math.BigDecimal.setScale(BigDecimal.java:2470)
   [junit4]    >        at 
java.math.BigDecimal.toBigInteger(BigDecimal.java:3025)
   [junit4]    >        at 
org.apache.lucene.util.TestUtil.nextLong(TestUtil.java:443)
   [junit4]    >        at 
org.apache.lucene.geo.GeoTestUtil.nextDoubleInternal(GeoTestUtil.java:71)
   [junit4]    >        at 
org.apache.lucene.geo.GeoTestUtil.nextLongitude(GeoTestUtil.java:40)
   [junit4]    >        at 
org.apache.lucene.geo.GeoTestUtil.nextBoxInternal(GeoTestUtil.java:423)
   [junit4]    >        at 
org.apache.lucene.geo.GeoTestUtil.nextPolygon(GeoTestUtil.java:405)
   [junit4]    >        at 
org.apache.lucene.document.TestLatLonShapeQueries.doTestRandom(TestLatLonShapeQueries.java:126)
   [junit4]    >        at 
org.apache.lucene.document.TestLatLonShapeQueries.testRandomBig(TestLatLonShapeQueries.java:106)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/sandbox/test/J0/temp/lucene.document.TestLatLonShapeQueries_3643D60F496A3CC6-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=RandomSimilarity(queryNorm=true): {}, locale=bg, timezone=PRC
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 
1.8.0_172 (64-bit)/cpus=4,threads=1,free=315734704,total=325058560
   [junit4]   2> NOTE: All tests run in this JVM: [TestNearest, 
FuzzyLikeThisQueryTest, TestLatLonShapeQueries]
   [junit4] Completed [18/19 (1!)] on J0 in 413.40s, 3 tests, 1 error <<< 
FAILURES!

[...truncated 4 lines...]
   [junit4] JVM J0: stdout was not empty, see: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/sandbox/test/temp/junit4-J0-20180726_124558_8001666320474710457122.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid9048.hprof
 ...
   [junit4] Heap dump file created [456422998 bytes in 1.106 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 3090 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderTragicEventTest
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/init-core-data-001
   [junit4]   2> 928257 WARN  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 928257 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 928259 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 928259 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 928260 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001
   [junit4]   2> 928260 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 928268 INFO  (Thread-633) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 928268 INFO  (Thread-633) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 928283 ERROR (Thread-633) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 928368 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:39022
   [junit4]   2> 928378 INFO  (zkConnectionManagerCallback-1815-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928400 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: 
d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 928402 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: 
d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 928404 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 928404 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 928404 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 928406 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 928406 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 928406 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 928410 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6a42ee8b{/solr,null,AVAILABLE}
   [junit4]   2> 928413 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@6859c655{SSL,[ssl, 
http/1.1]}{127.0.0.1:37732}
   [junit4]   2> 928413 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.e.j.s.Server Started @928615ms
   [junit4]   2> 928413 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=37732}
   [junit4]   2> 928413 ERROR (jetty-launcher-1812-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 928413 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.5.0
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2018-07-26T13:45:14.648Z
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@d460e48{/solr,null,AVAILABLE}
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@33408fdc{SSL,[ssl, 
http/1.1]}{127.0.0.1:46589}
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.e.j.s.Server Started @928616ms
   [junit4]   2> 928414 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=46589}
   [junit4]   2> 928415 ERROR (jetty-launcher-1812-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 928415 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 928415 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.5.0
   [junit4]   2> 928415 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 928415 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 928415 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2018-07-26T13:45:14.649Z
   [junit4]   2> 928428 INFO  (zkConnectionManagerCallback-1817-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928429 INFO  (zkConnectionManagerCallback-1819-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928429 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 928429 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 928515 INFO  (jetty-launcher-1812-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39022/solr
   [junit4]   2> 928535 INFO  (zkConnectionManagerCallback-1823-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928560 INFO  (zkConnectionManagerCallback-1825-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928592 INFO  (jetty-launcher-1812-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39022/solr
   [junit4]   2> 928601 INFO  (zkConnectionManagerCallback-1831-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928611 INFO  (zkConnectionManagerCallback-1833-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928623 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:46589_solr
   [junit4]   2> 928624 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.c.Overseer Overseer 
(id=73037364803731462-127.0.0.1:46589_solr-n_0000000000) starting
   [junit4]   2> 928632 INFO  (zkConnectionManagerCallback-1840-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928634 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:39022/solr ready
   [junit4]   2> 928641 INFO  
(OverseerStateUpdate-73037364803731462-127.0.0.1:46589_solr-n_0000000000) [    
] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46589_solr
   [junit4]   2> 928653 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:46589_solr
   [junit4]   2> 928657 INFO  (zkCallback-1832-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 928657 INFO  (zkCallback-1839-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 928689 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 928736 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46589.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928737 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 928741 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating 
transient cache for 2147483647 transient cores
   [junit4]   2> 928741 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:37732_solr
   [junit4]   2> 928742 INFO  (zkCallback-1839-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 928742 INFO  (zkCallback-1832-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 928761 INFO  (zkCallback-1824-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 928763 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46589.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928763 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46589.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928764 INFO  (jetty-launcher-1812-thread-1) 
[n:127.0.0.1:46589_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node2/.
   [junit4]   2> 928778 INFO  (zkConnectionManagerCallback-1846-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928779 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 928780 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:39022/solr ready
   [junit4]   2> 928780 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 928812 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_37732.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928831 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_37732.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928832 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_37732.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 928833 INFO  (jetty-launcher-1812-thread-2) 
[n:127.0.0.1:37732_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node1/.
   [junit4]   2> 928905 INFO  (zkConnectionManagerCallback-1849-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928940 INFO  (zkConnectionManagerCallback-1854-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 928941 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 928942 INFO  
(SUITE-LeaderTragicEventTest-seed#[14F869F052BC897B]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39022/solr ready
   [junit4]   2> 929105 INFO  
(TEST-LeaderTragicEventTest.test-seed#[14F869F052BC897B]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 929135 INFO  (qtp1925953728-5166) [n:127.0.0.1:37732_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
collection.configName=config&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 929169 INFO  (OverseerThreadFactory-1902-thread-1) [    ] 
o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 929321 INFO  
(OverseerStateUpdate-73037364803731462-127.0.0.1:46589_solr-n_0000000000) [    
] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:37732/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 929323 INFO  
(OverseerStateUpdate-73037364803731462-127.0.0.1:46589_solr-n_0000000000) [    
] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:46589/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 929586 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr    
x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 929596 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr    
x:collection1_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 929596 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr    
x:collection1_shard1_replica_n2] o.a.s.c.TransientSolrCoreCacheDefault 
Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 930614 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 930620 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 930627 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 930630 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 930630 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n2' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 930637 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 930650 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 930650 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 930654 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46589.solr.core.collection1.shard1.replica_n2' (registry 
'solr.core.collection1.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 930654 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 930654 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n2] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node2/collection1_shard1_replica_n2],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node2/./collection1_shard1_replica_n2/data/]
   [junit4]   2> 930659 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_37732.solr.core.collection1.shard1.replica_n1' (registry 
'solr.core.collection1.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@710a1eb6
   [junit4]   2> 930662 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 930662 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at 
[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node1/collection1_shard1_replica_n1],
 
dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/tempDir-001/node1/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 930761 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 930761 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 930762 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 930762 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 930769 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1a398c09[collection1_shard1_replica_n1] main]
   [junit4]   2> 930776 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 930790 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 930790 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 930790 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 930791 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 930792 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1607060912368254976
   [junit4]   2> 930794 INFO  
(searcherExecutor-1912-thread-1-processing-n:127.0.0.1:37732_solr 
x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) 
[n:127.0.0.1:37732_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SolrCore 
[collection1_shard1_replica_n1] Registered new searcher 
Searcher@1a398c09[collection1_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 930795 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 930795 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 930803 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1cc16c65[collection1_shard1_replica_n2] main]
   [junit4]   2> 930805 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 930806 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 930808 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 930810 INFO  
(searcherExecutor-1911-thread-1-processing-n:127.0.0.1:46589_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:46589_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2] Registered new searcher 
Searcher@1cc16c65[collection1_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 930810 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 930811 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1607060912388177920
   [junit4]   2> 930818 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 930824 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node3=0, 
core_node4=0}, version=1}
   [junit4]   2> 931319 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 931319 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 931319 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/
   [junit4]   2> 931319 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n1 
url=https://127.0.0.1:37732/solr START 
replicas=[https://127.0.0.1:46589/solr/collection1_shard1_replica_n2/] 
nUpdates=100
   [junit4]   2> 931320 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n1 
url=https://127.0.0.1:37732/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 931363 INFO  (qtp1615288217-5175) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.c.S.Request [collection1_shard1_replica_n2]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=7
   [junit4]   2> 931363 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the 
next candidate
   [junit4]   2> 931363 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 931363 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 931367 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 931469 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 931471 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1886
   [junit4]   2> 931572 INFO  (zkCallback-1824-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 931829 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_n2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=2233
   [junit4]   2> 931835 INFO  (qtp1925953728-5166) [n:127.0.0.1:37732_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 931930 INFO  (zkCallback-1824-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 931930 INFO  (zkCallback-1832-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 932836 INFO  (qtp1925953728-5166) [n:127.0.0.1:37732_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={collection.configName=config&name=collection1&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=3700
   [junit4]   2> 932840 INFO  
(TEST-LeaderTragicEventTest.test-seed#[14F869F052BC897B]) [    ] 
o.a.s.c.LeaderTragicEventTest Corrupt leader : 
core_node3:{"core":"collection1_shard1_replica_n1","base_url":"https://127.0.0.1:37732/solr","node_name":"127.0.0.1:37732_solr","state":"active","type":"NRT","force_set_state":"false","leader":"true"}
   [junit4]   2> 932915 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/collection1/terms/shard1 to Terms{values={core_node3=1, 
core_node4=1}, version=2}
   [junit4]   2> 932926 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{add=[0
 (1607060914576556032)]} 0 1
   [junit4]   2> 932927 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 
(1607060914576556032)]} 0 28
   [junit4]   2> 932948 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1607060914628984832,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 932953 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@36c5aff4 
commitCommandVersion:1607060914628984832
   [junit4]   2> 932958 INFO  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1607060914639470592,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 932969 INFO  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@7bf15a25 
commitCommandVersion:1607060914639470592
   [junit4]   2> 933034 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@4be3bb01[collection1_shard1_replica_n2] main]
   [junit4]   2> 933036 INFO  
(searcherExecutor-1911-thread-1-processing-n:127.0.0.1:46589_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:46589_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2] Registered new searcher 
Searcher@4be3bb01[collection1_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.5.0):C1)))}
   [junit4]   2> 933036 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 933036 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 87
   [junit4]   2> 933062 INFO  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@28da7f55[collection1_shard1_replica_n1] main]
   [junit4]   2> 933062 INFO  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 933063 INFO  
(searcherExecutor-1912-thread-1-processing-n:127.0.0.1:37732_solr 
x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) 
[n:127.0.0.1:37732_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.c.SolrCore 
[collection1_shard1_replica_n1] Registered new searcher 
Searcher@28da7f55[collection1_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.5.0):C1)))}
   [junit4]   2> 933063 INFO  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 105
   [junit4]   2> 933064 INFO  (qtp1925953728-5172) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update 
params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
 0 136
   [junit4]   2> 933076 INFO  (qtp1615288217-5171) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{add=[1
 (1607060914757959680)]} 0 1
   [junit4]   2> 933077 INFO  (qtp1925953728-5178) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 
(1607060914757959680)]} 0 6
   [junit4]   2> 933078 INFO  (qtp1925953728-5170) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1607060914765299712,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 933078 INFO  (qtp1925953728-5170) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@7bf15a25 
commitCommandVersion:1607060914765299712
   [junit4]   2> 933081 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1607060914768445440,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 933081 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@36c5aff4 
commitCommandVersion:1607060914768445440
   [junit4]   2> 933115 INFO  (qtp1925953728-5170) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{}
 0 37
   [junit4]   2> 933116 ERROR (qtp1925953728-5170) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.h.RequestHandlerBase java.nio.file.NoSuchFileException: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/index-NIOFSDirectory-002/_0.fdx
   [junit4]   2>        at 
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]   2>        at 
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]   2>        at 
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]   2>        at 
sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
   [junit4]   2>        at 
org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]   2>        at 
org.apache.lucene.mockfile.DisableFsyncFS.newFileChannel(DisableFsyncFS.java:45)
   [junit4]   2>        at 
org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
java.nio.channels.FileChannel.open(FileChannel.java:287)
   [junit4]   2>        at 
java.nio.channels.FileChannel.open(FileChannel.java:335)
   [junit4]   2>        at 
org.apache.lucene.util.IOUtils.fsync(IOUtils.java:460)
   [junit4]   2>        at 
org.apache.lucene.store.FSDirectory.fsync(FSDirectory.java:332)
   [junit4]   2>        at 
org.apache.lucene.store.FSDirectory.sync(FSDirectory.java:287)
   [junit4]   2>        at 
org.apache.lucene.store.MockDirectoryWrapper.sync(MockDirectoryWrapper.java:207)
   [junit4]   2>        at 
org.apache.lucene.store.LockValidatingDirectoryWrapper.sync(LockValidatingDirectoryWrapper.java:68)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4782)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3282)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3450)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3415)
   [junit4]   2>        at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:671)
   [junit4]   2>        at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
   [junit4]   2>        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
   [junit4]   2>        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1940)
   [junit4]   2>        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1916)
   [junit4]   2>        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69)
   [junit4]   2>        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.execute(SolrCore.java:2541)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:674)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:531)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:291)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:151)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
   [junit4]   2>        at 
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 933117 ERROR (qtp1925953728-5170) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.s.HttpSolrCall null:java.nio.file.NoSuchFileException: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/index-NIOFSDirectory-002/_0.fdx
   [junit4]   2>        at 
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
   [junit4]   2>        at 
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
   [junit4]   2>        at 
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
   [junit4]   2>        at 
sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
   [junit4]   2>        at 
org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]   2>        at 
org.apache.lucene.mockfile.DisableFsyncFS.newFileChannel(DisableFsyncFS.java:45)
   [junit4]   2>        at 
org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
   [junit4]   2>        at 
java.nio.channels.FileChannel.open(FileChannel.java:287)
   [junit4]   2>        at 
java.nio.channels.FileChannel.open(FileChannel.java:335)
   [junit4]   2>        at 
org.apache.lucene.util.IOUtils.fsync(IOUtils.java:460)
   [junit4]   2>        at 
org.apache.lucene.store.FSDirectory.fsync(FSDirectory.java:332)
   [junit4]   2>        at 
org.apache.lucene.store.FSDirectory.sync(FSDirectory.java:287)
   [junit4]   2>        at 
org.apache.lucene.store.MockDirectoryWrapper.sync(MockDirectoryWrapper.java:207)
   [junit4]   2>        at 
org.apache.lucene.store.LockValidatingDirectoryWrapper.sync(LockValidatingDirectoryWrapper.java:68)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4782)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3282)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3450)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3415)
   [junit4]   2>        at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:671)
   [junit4]   2>        at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
   [junit4]   2>        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
   [junit4]   2>        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1940)
   [junit4]   2>        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1916)
   [junit4]   2>        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69)
   [junit4]   2>        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.execute(SolrCore.java:2541)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:674)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:531)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:291)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:151)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
   [junit4]   2>        at 
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 933118 ERROR 
(updateExecutor-1820-thread-1-processing-n:127.0.0.1:37732_solr 
x:collection1_shard1_replica_n1 c:collection1 s:shard1 r:core_node3) 
[n:127.0.0.1:37732_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_n1] o.a.s.u.SolrCmdDistributor 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:37732/solr/collection1_shard1_replica_n1: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/index-NIOFSDirectory-002/_0.fdx
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.request(ConcurrentUpdateSolrClient.java:489)
   [junit4]   2>        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2>        at 
org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:312)
   [junit4]   2>        at 
org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:301)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 933128 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@3d4b05d5[collection1_shard1_replica_n2] main]
   [junit4]   2> 933130 INFO  
(searcherExecutor-1911-thread-1-processing-n:127.0.0.1:46589_solr 
x:collection1_shard1_replica_n2 c:collection1 s:shard1 r:core_node4) 
[n:127.0.0.1:46589_solr c:collection1 s:shard1 r:core_node4 
x:collection1_shard1_replica_n2] o.a.s.c.SolrCore 
[collection1_shard1_replica_n2] Registered new searcher 
Searcher@3d4b05d5[collection1_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.5.0):C1)
 Uninverting(_1(7.5.0):C1)))}
   [junit4]   2> 933130 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 933130 INFO  (qtp1615288217-5174) [n:127.0.0.1:46589_solr 
c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:37732/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 49
   [junit4]   2> 933131 WARN  (qtp1925953728-5177) [n:127.0.0.1:37732_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
https://127.0.0.1:37732/solr
   [junit4]   2> 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:37732/solr/collection1_shard1_replica_n1: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderTragicEventTest_14F869F052BC897B-001/index-NIOFSDirectory-002/_0.fdx
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
 ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
 ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
 ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.request(ConcurrentUpdateSolrClient.java:489)
 ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:312)
 ~[java/:?]
   [junit4]   2>        at 
org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:301)
 ~[java/:?]
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
~[?:1.8.0_172]
   [junit4]   2>        at java.util.concurrent.Fut

[...truncated too long message...]

taNode Ending block pool service for: Block pool 
BP-1749227281-127.0.0.1-1532616793034 (Datanode Uuid 
a5c11042-07e4-4723-ba74-b26add397d65) service to 
lucene2-us-west.apache.org/127.0.0.1:42670
   [junit4]   2> 5271595 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[14F869F052BC897B]-worker) [    ] 
o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5271683 WARN  (1754100950@qtp-1039239134-1 - Acceptor0 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40563) [    ] 
o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
   [junit4]   2> 5271683 WARN  (1754100950@qtp-1039239134-1 - Acceptor0 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40563) [    ] 
o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
   [junit4]   2> 5271721 WARN  
(SUITE-HdfsBasicDistributedZk2Test-seed#[14F869F052BC897B]-worker) [    ] 
o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 5272120 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[14F869F052BC897B]-worker) [    ] 
o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 5272246 WARN  (DataNode: 
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_14F869F052BC897B-001/tempDir-001/hdfsBaseDir/data/data1/,
 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_14F869F052BC897B-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to lucene2-us-west.apache.org/127.0.0.1:42670) [    ] 
o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager 
interrupted
   [junit4]   2> 5272246 WARN  (DataNode: 
[[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_14F869F052BC897B-001/tempDir-001/hdfsBaseDir/data/data1/,
 
[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_14F869F052BC897B-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to lucene2-us-west.apache.org/127.0.0.1:42670) [    ] 
o.a.h.h.s.d.DataNode Ending block pool service for: Block pool 
BP-1749227281-127.0.0.1-1532616793034 (Datanode Uuid 
873d62c0-aa46-486f-a0b3-5848923677c4) service to 
lucene2-us-west.apache.org/127.0.0.1:42670
   [junit4]   2> 5272428 INFO  
(SUITE-HdfsBasicDistributedZk2Test-seed#[14F869F052BC897B]-worker) [    ] 
o.m.log Stopped 
HttpServer2$selectchannelconnectorwithsafestar...@lucene2-us-west.apache.org:0
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_14F869F052BC897B-001
   [junit4]   2> Jul 26, 2018 2:57:39 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 18 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{rnd_b=PostingsFormat(name=LuceneFixedGap), 
SubjectTerms_mfacet=PostingsFormat(name=Asserting), 
oddField_s=PostingsFormat(name=LuceneFixedGap), 
multiDefault=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), 
foo_b=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
 id=PostingsFormat(name=LuceneFixedGap), 
text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))},
 docValues:{other_tl1=DocValuesFormat(name=Memory), 
range_facet_l_dv=DocValuesFormat(name=Lucene70), 
n_l1=DocValuesFormat(name=Lucene70), 
multiDefault=DocValuesFormat(name=Lucene70), 
intDefault=DocValuesFormat(name=Memory), n_dt1=DocValuesFormat(name=Memory), 
a_i1=DocValuesFormat(name=Lucene70), n_td1=DocValuesFormat(name=Memory), 
n_d1=DocValuesFormat(name=Lucene70), 
range_facet_l=DocValuesFormat(name=Lucene70), 
n_f1=DocValuesFormat(name=Memory), n_ti1=DocValuesFormat(name=Lucene70), 
n_tl1=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Memory), 
n_tf1=DocValuesFormat(name=Lucene70), n_tdt1=DocValuesFormat(name=Memory), 
id_i1=DocValuesFormat(name=Memory), foo_d=DocValuesFormat(name=Lucene70), 
range_facet_i_dv=DocValuesFormat(name=Lucene70), 
id=DocValuesFormat(name=Lucene70), 
intDvoDefault=DocValuesFormat(name=Lucene70), 
foo_f=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Lucene70)}, 
maxPointsInLeafNode=1892, maxMBSortInHeap=7.390212610132505, 
sim=RandomSimilarity(queryNorm=false): {}, locale=be-BY, timezone=Asia/Rangoon
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 
1.8.0_172 (64-bit)/cpus=4,threads=2,free=90255768,total=517472256
   [junit4]   2> NOTE: All tests run in this JVM: 
[ScheduledMaintenanceTriggerTest, ResponseLogComponentTest, TestRealTimeGet, 
AnalyticsQueryTest, URLClassifyProcessorTest, ZkFailoverTest, 
TestAuthorizationFramework, ZkSolrClientTest, 
OverseerCollectionConfigSetProcessorTest, ShardSplitTest, TestFastLRUCache, 
SuggesterFSTTest, TestFieldTypeCollectionResource, SSLMigrationTest, 
ConcurrentCreateRoutedAliasTest, CoreAdminHandlerTest, 
CdcrReplicationDistributedZkTest, DistributedFacetPivotLongTailTest, 
TestConfigsApi, TestJsonFacetRefinement, TestBulkSchemaAPI, TestReload, 
TestInitParams, ResponseHeaderTest, TestManagedSchema, TestReplicationHandler, 
LargeFieldTest, TestSchemaResource, TestNoOpRegenerator, TestHashQParserPlugin, 
TestPayloadScoreQParserPlugin, TestCustomStream, HdfsRestartWhileUpdatingTest, 
SampleTest, LeaderFailureAfterFreshStartTest, BaseCdcrDistributedZkTest, 
BigEndianAscendingWordDeserializerTest, FieldMutatingUpdateProcessorTest, 
SegmentsInfoRequestHandlerTest, OpenExchangeRatesOrgProviderTest, 
DocValuesTest, TestInPlaceUpdatesStandalone, UpdateParamsTest, 
TestRetrieveFieldsOptimizer, HLLUtilTest, QueryParsingTest, SolrCLIZkUtilsTest, 
SuggestComponentTest, CreateCollectionCleanupTest, PeerSyncReplicationTest, 
TestRangeQuery, ScriptEngineTest, RandomizedTaggerTest, TestDynamicURP, 
TestConfigSetImmutable, TestSchemalessBufferedUpdates, 
TestLMDirichletSimilarityFactory, SpellCheckCollatorTest, TaggingAttributeTest, 
TestQuerySenderNoQuery, ChaosMonkeyNothingIsSafeTest, 
TestBlendedInfixSuggestions, DistributedFacetSimpleRefinementLongTailTest, 
AutoAddReplicasPlanActionTest, SystemInfoHandlerTest, 
CloudExitableDirectoryReaderTest, ResourceLoaderTest, TestSafeXMLParsing, 
TestManagedSynonymFilterFactory, SpatialRPTFieldTypeTest, TestExportWriter, 
TestUseDocValuesAsStored2, TestGroupingSearch, QueryResultKeyTest, 
DateRangeFieldTest, TestSolrCloudWithSecureImpersonation, 
TestPullReplicaErrorHandling, SchemaApiFailureTest, TestSolrConfigHandlerCloud, 
DirectUpdateHandlerTest, RulesTest, UtilsToolTest, TestHttpShardHandlerFactory, 
TestLuceneMatchVersion, DefaultValueUpdateProcessorTest, 
AnalysisErrorHandlingTest, TestGraphMLResponseWriter, TestBulkSchemaConcurrent, 
TestCloudDeleteByQuery, TestCollationField, TestMinMaxOnMultiValuedField, 
ParsingFieldUpdateProcessorsTest, DeleteLastCustomShardedReplicaTest, 
CursorPagingTest, ZkStateWriterTest, AtomicUpdateProcessorFactoryTest, 
SimpleMLTQParserTest, TestSolrFieldCacheBean, CoreSorterTest, 
TestSegmentSorting, CopyFieldTest, UninvertDocValuesMergePolicyTest, 
TestCodecSupport, TestDynamicFieldResource, TestSort, 
CustomHighlightComponentTest, TestPostingsSolrHighlighter, 
BasicDistributedZk2Test, LukeRequestHandlerTest, 
PathHierarchyTokenizerFactoryTest, 
PeerSyncWithLeaderAndIndexFingerprintCachingTest, SpellCheckComponentTest, 
TestRemoteStreaming, SimplePostToolTest, TestHashPartitioner, 
HdfsNNFailoverTest, TestValueSourceCache, TestCloudRecovery, MoveReplicaTest, 
TestOmitPositions, TestWriterPerf, ChaosMonkeySafeLeaderWithPullReplicasTest, 
ExplicitHLLTest, TestCrossCoreJoin, TokenizerChainTest, ShardRoutingTest, 
TestMergePolicyConfig, TestRestoreCore, TestSubQueryTransformer, 
SolrJmxReporterCloudTest, TestEmbeddedSolrServerConstructors, 
SolrCoreMetricManagerTest, TestStressRecovery, TestStressLiveNodes, 
LIRRollingUpdatesTest, LeaderElectionTest, MigrateRouteKeyTest, 
MoveReplicaHDFSTest, TestAuthenticationFramework, TestCloudConsistency, 
TestConfigSetsAPI, TestLeaderInitiatedRecoveryThread, TestLockTree, 
TestMiniSolrCloudClusterSSL, TestRequestForwarding, TestSSLRandomization, 
TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TestZkChroot, 
TlogReplayBufferedWhileIndexingTest, CollectionTooManyReplicasTest, 
CustomCollectionTest, HdfsCollectionsAPIDistributedZkTest, 
TestCollectionsAPIViaSolrCloudCluster, TestLocalFSCloudBackupRestore, 
TestReplicaProperties, HdfsAutoAddReplicasIntegrationTest, 
HttpTriggerListenerTest, MetricTriggerTest, NodeAddedTriggerIntegrationTest, 
NodeLostTriggerIntegrationTest, NodeMarkersRegistrationTest, 
SearchRateTriggerTest, TestPolicyCloud, TestClusterStateProvider, 
TestComputePlanAction, TestDistribStateManager, TestExecutePlanAction, 
TestGenericDistributedQueue, TestLargeCluster, TestNodeAddedTrigger, 
TestNodeLostTrigger, TestPolicyCloud, TestTriggerIntegration, 
CdcrVersionReplicationTest, HdfsBasicDistributedZk2Test]
   [junit4] Completed [564/825 (3!)] on J0 in 277.00s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 9682 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651:
 The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585:
 Some of the tests produced a heap dump, but did not fail. Maybe a suppressed 
OutOfMemoryError? Dumps created:
* java_pid9048.hprof

Total time: 448 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to