[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1350 - Failure

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1350/

9 tests failed.
FAILED:  org.apache.lucene.codecs.lucene70.TestIndexedDISI.testRandom

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  
org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit

Error Message:
this IndexWriter is closed

Stack Trace:
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
        at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
        at 
org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:230)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
        at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
        at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)


FAILED:  org.apache.lucene.search.TestDoubleRangeFieldQueries.testRandomBig

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space


FAILED:  org.apache.lucene.util.TestPagedBytes.testDataInputOutput2

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6:8B5FCA82AA91D006]:0)
        at java.util.Arrays.copyOf(Arrays.java:3181)
        at org.apache.lucene.util.PagedBytes.addBlock(PagedBytes.java:158)
        at org.apache.lucene.util.PagedBytes.access$900(PagedBytes.java:36)
        at 
org.apache.lucene.util.PagedBytes$PagedBytesDataOutput.writeBytes(PagedBytes.java:390)
        at 
org.apache.lucene.util.TestPagedBytes.testDataInputOutput2(TestPagedBytes.java:116)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)


FAILED:  org.apache.lucene.util.TestPagedBytes.testDataInputOutput

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6:5ED01300913B26B5]:0)
        at java.util.Arrays.copyOf(Arrays.java:3181)
        at org.apache.lucene.util.PagedBytes.addBlock(PagedBytes.java:158)
        at org.apache.lucene.util.PagedBytes.copy(PagedBytes.java:169)
        at 
org.apache.lucene.util.TestPagedBytes.testDataInputOutput(TestPagedBytes.java:66)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
10 threads leaked from SUITE scope at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:     1) 
Thread[id=30820, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-EventThread,
 state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]        
 at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    
2) Thread[id=30818, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=30897, 
name=StoppableCommitThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)  
  4) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]   
      at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    
5) Thread[id=30924, name=StoppableCommitThread-EventThread, state=WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    
6) Thread[id=30892, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
    7) Thread[id=30894, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
    8) Thread[id=30893, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
    9) Thread[id=30895, name=StoppableSearchThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:65)  
 10) Thread[id=30819, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]  
       at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from 
SUITE scope at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 
   1) Thread[id=30820, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-EventThread,
 state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=30818, name=Connection evictor, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=30897, name=StoppableCommitThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
   4) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   5) Thread[id=30924, name=StoppableCommitThread-EventThread, state=WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   6) Thread[id=30892, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   7) Thread[id=30894, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   8) Thread[id=30893, name=StoppableIndexingThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
   9) Thread[id=30895, name=StoppableSearchThread, state=TIMED_WAITING, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:65)
  10) Thread[id=30819, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]   
      at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    
2) Thread[id=30892, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)     
    at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
         at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)       
  at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
    3) Thread[id=30894, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)     
    at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
         at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)       
  at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
    4) Thread[id=30893, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90)         at 
org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840)        
 at 
org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)   
      at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)  
       at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)        
 at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)     
    at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
         at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)       
  at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
    5) Thread[id=30895, name=StoppableSearchThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]         at 
java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at 
java.io.PrintStream.flush(PrintStream.java:338)         at 
sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)         at 
sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)         at 
java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)         at 
org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)         at 
org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)         at 
org.apache.log4j.WriterAppender.append(WriterAppender.java:162)         at 
org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)         
at 
org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
         at org.apache.log4j.Category.callAppenders(Category.java:206)         
at org.apache.log4j.Category.forcedLog(Category.java:391)         at 
org.apache.log4j.Category.log(Category.java:856)         at 
org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)         at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897)
         at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)         
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)         
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)         
at 
org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)  
  6) Thread[id=30819, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]  
       at java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), 
state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   2) Thread[id=30892, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
        at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   3) Thread[id=30894, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
        at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   4) Thread[id=30893, name=StoppableIndexingThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90)
        at 
org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840)
        at 
org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
        at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
        at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
        at 
org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
        at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
        at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
        at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
        at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   5) Thread[id=30895, name=StoppableSearchThread, state=RUNNABLE, 
group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
        at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
        at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
        at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
        at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
        at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
        at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
        at 
org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
        at org.apache.log4j.Category.callAppenders(Category.java:206)
        at org.apache.log4j.Category.forcedLog(Category.java:391)
        at org.apache.log4j.Category.log(Category.java:856)
        at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at 
org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
   6) Thread[id=30819, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
        at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth 
investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties 
were stopped - this is worth investigating!
        at 
__randomizedtesting.SeedInfo.seed([243C1376CCE7A90E:AC682CAC621BC4F6]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
        at 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:222)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 898 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexingSequenceNumbers
   [junit4]   2> Jul 09, 2017 10:03:03 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1072,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: java.lang.IllegalStateException: 
this writer hit an unrecoverable error; cannot commit
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: java.lang.IllegalStateException: this writer hit an 
unrecoverable error; cannot commit
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4689)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3279)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:21 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1071,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:18 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1078,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:13 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1069,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:08 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1075,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:08 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1073,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:38 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1077,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:38 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1074,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:03:13 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1070,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:14:13 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge 
Thread #1735,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: 
java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:703)
   [junit4]   2>        at 
org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:683)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at java.util.Arrays.copyOfRange(Arrays.java:3664)
   [junit4]   2>        at java.lang.String.<init>(String.java:207)
   [junit4]   2>        at 
java.lang.StringBuilder.toString(StringBuilder.java:407)
   [junit4]   2>        at 
org.apache.lucene.store.BufferedChecksumIndexInput.<init>(BufferedChecksumIndexInput.java:34)
   [junit4]   2>        at 
org.apache.lucene.codecs.CodecUtil.checksumEntireFile(CodecUtil.java:520)
   [junit4]   2>        at 
org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.checkIntegrity(Lucene50PostingsReader.java:1283)
   [junit4]   2>        at 
org.apache.lucene.codecs.memory.FSTOrdTermsReader.checkIntegrity(FSTOrdTermsReader.java:875)
   [junit4]   2>        at 
org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.checkIntegrity(PerFieldPostingsFormat.java:348)
   [junit4]   2>        at 
org.apache.lucene.index.CodecReader.checkIntegrity(CodecReader.java:286)
   [junit4]   2>        at 
org.apache.lucene.index.MergeReaderWrapper.checkIntegrity(MergeReaderWrapper.java:178)
   [junit4]   2>        at 
org.apache.lucene.index.FilterLeafReader.checkIntegrity(FilterLeafReader.java:408)
   [junit4]   2>        at 
org.apache.lucene.index.SlowCodecReaderWrapper.wrap(SlowCodecReaderWrapper.java:54)
   [junit4]   2>        at 
org.apache.lucene.index.MockRandomMergePolicy$MockRandomOneMerge.wrapForMerge(MockRandomMergePolicy.java:179)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4375)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
   [junit4]   2>        at 
org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
   [junit4]   2>        at 
org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
   [junit4]   2> 
   [junit4]   2> Jul 09, 2017 10:14:13 AM 
com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler
 uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: 
Thread[Thread-1076,5,TGRP-TestIndexingSequenceNumbers]
   [junit4]   2> java.lang.OutOfMemoryError: Java heap space
   [junit4]   2>        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]   2>        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]   2>        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]   2>        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]   2>        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]   2>        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]   2>        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
   [junit4]   2> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestIndexingSequenceNumbers 
-Dtests.method=testStressConcurrentCommit -Dtests.seed=59EC404FA001EC6 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=de-CH -Dtests.timezone=Etc/GMT -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   1630s J0 | 
TestIndexingSequenceNumbers.testStressConcurrentCommit <<<
   [junit4]    > Throwable #1: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:230)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1215, name=Thread-1072, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot 
commit
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: java.lang.IllegalStateException: this writer hit an 
unrecoverable error; cannot commit
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4689)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3279)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1216, name=Thread-1073, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1218, name=Thread-1075, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1212, name=Thread-1069, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1213, name=Thread-1070, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1221, name=Thread-1078, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #8: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1220, name=Thread-1077, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable
 #9: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an 
uncaught exception in thread: Thread[id=1214, name=Thread-1071, state=RUNNABLE, 
group=TGRP-TestIndexingSequenceNumbers]
   [junit4]    > Caused by: java.lang.RuntimeException: 
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: 
this IndexWriter is closed
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
   [junit4]    >        at 
org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
   [junit4]    > Caused by: java.lang.OutOfMemoryError: Java heap space
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
   [junit4]    >        at 
org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
   [junit4]    >        at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    >        at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
   [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.proce

[...truncated too long message...]

a:241)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]    >    3) Thread[id=30894, name=StoppableIndexingThread, 
state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]    >    4) Thread[id=30893, name=StoppableIndexingThread, 
state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
   [junit4]    >         at 
org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
   [junit4]    >         at 
org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
   [junit4]    >         at 
org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
   [junit4]    >    5) Thread[id=30895, name=StoppableSearchThread, 
state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
   [junit4]    >         at 
java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
   [junit4]    >         at java.io.PrintStream.flush(PrintStream.java:338)
   [junit4]    >         at 
sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
   [junit4]    >         at 
sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
   [junit4]    >         at 
java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
   [junit4]    >         at 
org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
   [junit4]    >         at 
org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
   [junit4]    >         at 
org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
   [junit4]    >         at 
org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
   [junit4]    >         at 
org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
   [junit4]    >         at 
org.apache.log4j.Category.callAppenders(Category.java:206)
   [junit4]    >         at 
org.apache.log4j.Category.forcedLog(Category.java:391)
   [junit4]    >         at org.apache.log4j.Category.log(Category.java:856)
   [junit4]    >         at 
org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897)
   [junit4]    >         at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
   [junit4]    >         at 
org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
   [junit4]    >         at 
org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
   [junit4]    >    6) Thread[id=30819, 
name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987),
 state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)
   [junit4] Completed [502/727 (1!)] on J2 in 122.15s, 1 test, 1 failure, 2 
errors <<< FAILURES!

[...truncated 8199 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:762:
 Some of the tests produced a heap dump, but did not fail. Maybe a suppressed 
OutOfMemoryError? Dumps created:
* java_pid28178.hprof

Total time: 263 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org