Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1436/

10 tests failed.
FAILED:  org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates

Error Message:
_1_b.fnm

Stack Trace:
java.io.FileNotFoundException: _1_b.fnm
        at 
__randomizedtesting.SeedInfo.seed([5800092A3F8C2B60:2025D721DDAC0482]:0)
        at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
        at 
org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
        at 
org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates

Error Message:
_1_c.fnm

Stack Trace:
java.io.FileNotFoundException: _1_c.fnm
        at 
__randomizedtesting.SeedInfo.seed([5800092A3F8C2B60:2025D721DDAC0482]:0)
        at org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
        at 
org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
        at 
org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates(TestMixedDocValuesUpdates.java:374)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat.testGlobalVersions

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
        at __randomizedtesting.SeedInfo.seed([504E9AE51F4C6630]:0)


FAILED:  
junit.framework.TestSuite.org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
        at __randomizedtesting.SeedInfo.seed([504E9AE51F4C6630]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest:   
  1) Thread[id=5286, name=zkConnectionManagerCallback-775-thread-1, 
state=WAITING, group=TGRP-LeaderElectionTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.cloud.LeaderElectionTest: 
   1) Thread[id=5286, name=zkConnectionManagerCallback-775-thread-1, 
state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([87653ED116AB22A0]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=5286, name=zkConnectionManagerCallback-775-thread-1, state=WAITING, 
group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)   
      at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)      
   at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=5286, name=zkConnectionManagerCallback-775-thread-1, 
state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([87653ED116AB22A0]:0)


FAILED:  org.apache.solr.cloud.TestHdfsCloudBackupRestore.test

Error Message:
expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
        at 
__randomizedtesting.SeedInfo.seed([87653ED116AB22A0:F31010BB8574F58]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:235)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:140)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 600 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 600 
seconds
        at 
__randomizedtesting.SeedInfo.seed([87653ED116AB22A0:F31010BB8574F58]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:913)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1470)
        at 
org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.uninverting.TestLegacyFieldCache

Error Message:
The test or suite printed 134160 bytes to stdout and stderr, even though the 
limit was set to 8192 bytes. Increase the limit with @Limit, ignore it 
completely with @SuppressSysoutChecks or run with -Dtests.verbose=true

Stack Trace:
java.lang.AssertionError: The test or suite printed 134160 bytes to stdout and 
stderr, even though the limit was set to 8192 bytes. Increase the limit with 
@Limit, ignore it completely with @SuppressSysoutChecks or run with 
-Dtests.verbose=true
        at __randomizedtesting.SeedInfo.seed([87653ED116AB22A0]:0)
        at 
org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:211)
        at 
com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.util.DateMathParserTest

Error Message:
The test or suite printed 11184 bytes to stdout and stderr, even though the 
limit was set to 8192 bytes. Increase the limit with @Limit, ignore it 
completely with @SuppressSysoutChecks or run with -Dtests.verbose=true

Stack Trace:
java.lang.AssertionError: The test or suite printed 11184 bytes to stdout and 
stderr, even though the limit was set to 8192 bytes. Increase the limit with 
@Limit, ignore it completely with @SuppressSysoutChecks or run with 
-Dtests.verbose=true
        at __randomizedtesting.SeedInfo.seed([87653ED116AB22A0]:0)
        at 
org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:211)
        at 
com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 404 lines...]
   [junit4] Suite: org.apache.lucene.index.TestBinaryDocValuesUpdates
   [junit4]   1> TEST: isNRT=true 
reader1=StandardDirectoryReader(segments:3:nrt _0(8.0.0):C2)
   [junit4]   1> TEST: now reopen
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestBinaryDocValuesUpdates -Dtests.method=testTonsOfUpdates 
-Dtests.seed=5800092A3F8C2B60 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=fr -Dtests.timezone=Etc/UCT -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   64.6s J0 | TestBinaryDocValuesUpdates.testTonsOfUpdates <<<
   [junit4]    > Throwable #1: java.io.FileNotFoundException: _1_b.fnm
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([5800092A3F8C2B60:2025D721DDAC0482]:0)
   [junit4]    >        at 
org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
   [junit4]    >        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]    >        at 
org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
   [junit4]    >        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
   [junit4]    >        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
   [junit4]    >        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
   [junit4]    >        at 
org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0/temp/lucene.index.TestBinaryDocValuesUpdates_5800092A3F8C2B60-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4efa365c),
 locale=fr, timezone=Etc/UCT
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=185586872,total=372768768
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestSimpleExplanationsWithFillerDocs, TestBoolean2ScorerSupplier, 
TestAxiomaticF3LOG, TestBytesRef, TestAllFilesCheckIndexHeader, 
TestDirectPacked, TestStressIndexing, TestNativeFSLockFactory, 
TestMultiTermsEnum, TestFilterCodecReader, TestIndexWriterThreadsToSegments, 
TestBinaryDocValuesUpdates]
   [junit4] Completed [42/475 (1!)] on J0 in 175.08s, 29 tests, 1 error <<< 
FAILURES!

[...truncated 1705 lines...]
   [junit4] Suite: org.apache.lucene.index.TestMixedDocValuesUpdates
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestMixedDocValuesUpdates -Dtests.method=testTonsOfUpdates 
-Dtests.seed=5800092A3F8C2B60 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=th-TH-u-nu-thai-x-lvariant-TH -Dtests.timezone=SystemV/AST4ADT 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   35.8s J0 | TestMixedDocValuesUpdates.testTonsOfUpdates <<<
   [junit4]    > Throwable #1: java.io.FileNotFoundException: _1_c.fnm
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([5800092A3F8C2B60:2025D721DDAC0482]:0)
   [junit4]    >        at 
org.apache.lucene.store.RAMDirectory.openInput(RAMDirectory.java:243)
   [junit4]    >        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]    >        at 
org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:41)
   [junit4]    >        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
   [junit4]    >        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
   [junit4]    >        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
   [junit4]    >        at 
org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates(TestMixedDocValuesUpdates.java:374)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@60d68c80),
 locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=SystemV/AST4ADT
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=166726968,total=492830720
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestSimpleExplanationsWithFillerDocs, TestBoolean2ScorerSupplier, 
TestAxiomaticF3LOG, TestBytesRef, TestAllFilesCheckIndexHeader, 
TestDirectPacked, TestStressIndexing, TestNativeFSLockFactory, 
TestMultiTermsEnum, TestFilterCodecReader, TestIndexWriterThreadsToSegments, 
TestBinaryDocValuesUpdates, TestIndexWriterExceptions, 
TestFieldMaskingSpanQuery, TestCrashCausesCorruptIndex, TestTopFieldCollector, 
TestFlex, Test2BTerms, TestDateTools, TestMultiThreadTermVectors, 
TestMatchAllDocsQuery, TestBinaryTerms, TestTermVectors, 
TestIndexWriterForceMerge, TestIndexWriterConfig, TestTryDelete, 
TestDocValuesIndexing, TestOneMergeWrappingMergePolicy, TestLongBitSet, 
TestSpansEnum, TestSnapshotDeletionPolicy, TestSpanMultiTermQueryWrapper, 
TestSortedSetDocValues, TestNRTCachingDirectory, TestCachingCollector, 
TestSparseFixedBitDocIdSet, TestSmallFloat, TestMergePolicyWrapper, 
TestTermVectorsWriter, TestIndexInput, TestDeterminizeLexicon, TestUTF32ToUTF8, 
TestWANDScorer, TestPostingsOffsets, TestCharArrayMap, 
TestIndexWriterNRTIsCurrent, TestIndexWriterWithThreads, Test4GBStoredFields, 
TestFloatRangeFieldQueries, TestNeedsScores, TestSimilarityProvider, 
TestBasicModelG, TestBooleanMinShouldMatch, TestUnicodeUtil, 
TestGrowableByteArrayDataOutput, TestDocIdSetBuilder, TestIntRangeFieldQueries, 
TestSimpleSearchEquivalence, TestDocIDMerger, TestMultiTermQueryRewrites, 
TestRegexpRandom, TestExitableDirectoryReader, TestNot, TestSimpleExplanations, 
TestLucene50TermVectorsFormat, FiniteStringsIteratorTest, 
TestThreadedForceMerge, TestSegmentMerger, TestCharFilter, 
TestSimpleFSLockFactory, TestTerm, TestLucene50CompoundFormat, 
TestDistributionLL, TestConstantScoreQuery, TestNamedSPILoader, 
TestDeletionPolicy, TestBKD, TestOperations, TestIndexWriter, 
TestPerFieldDocValuesFormat, TestIndexWriterReader, TestFSTs, TestBytesStore, 
TestAddIndexes, TestMultiMMap, TestSearcherManager, TestBooleanOr, 
TestFlushByRamOrCountsPolicy, TestNRTThreads, TestIndexWriterCommit, 
TestDirectoryReaderReopen, TestTimeLimitingCollector, TestSearchWithThreads, 
TestSearchAfter, TestConcurrentMergeScheduler, TestLongPostings, 
TestIndexWriterUnicode, TestNRTReaderWithThreads, TestPerFieldPostingsFormat, 
TestConsistentFieldNumbers, TestWeakIdentityMap, TestBagOfPositions, 
TestOmitNorms, TestFixedBitSet, TestComplexExplanations, TestRegexpRandom2, 
TestPagedBytes, TestSegmentTermDocs, TestPerFieldPostingsFormat2, TestSort, 
TestMultiDocValues, TestToken, TestCustomSearcherSort, 
TestIndexWriterOnDiskFull, TestSpans, TestCustomNorms, TestSpanExplanations, 
TestFastCompressionMode, TestStressIndexing2, TestMultiPhraseQuery, 
TestLazyProxSkipping, TestDeterminism, TestDocsAndPositions, 
TestMergeSchedulerExternal, TestBagOfPostings, TestFuzzyQuery, TestWildcard, 
TestSimilarity2, TestNoDeletionPolicy, TestRegexpQuery, TestTermVectorsReader, 
TestPerSegmentDeletes, TestIntBlockPool, TestTopDocsCollector, 
TestSegmentTermEnum, TestUniqueTermCount, TestAutomatonQuery, TestDocument, 
TestMaxTermFrequency, TestSumDocFreq, TestIndexableField, TestWildcardRandom, 
TestPrefixRandom, TestDirectory, TestCheckIndex, 
TestBooleanQueryVisitSubscorers, TestSizeBoundedForceMerge, 
TestIndexFileDeleter, TestCompiledAutomaton, TestParallelTermEnum, TestField, 
TestSimilarity, TestVirtualMethod, TestSetOnce, TestFilterLeafReader, 
TestSearch, TestCachingTokenFilter, TestFilterIterator, TestPriorityQueue, 
Test2BPostings, TestIndexWriterDeleteByQuery, TestIndexWriterMaxDocs, 
TestIndexWriterOutOfFileDescriptors, TestLogMergePolicy, TestManyFields, 
TestMaxPosition, TestMixedDocValuesUpdates]
   [junit4] Completed [391/475 (2!)] on J0 in 71.69s, 4 tests, 1 error <<< 
FAILURES!

[...truncated 8317 lines...]
   [junit4] Suite: 
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat
   [junit4]   2> dic 14, 2017 4:42:54 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: 
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat
   [junit4]   2>    1) Thread[id=87, 
name=SUITE-TestIDVersionPostingsFormat-seed#[504E9AE51F4C6630], state=RUNNABLE, 
group=TGRP-TestIDVersionPostingsFormat]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1559)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native 
Method)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:705)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:139)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:626)
   [junit4]   2>    2) Thread[id=9, name=JUnit4-serializer-daemon, 
state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:636)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:493)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
   [junit4]   2>         at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2>    4) Thread[id=88, 
name=TEST-TestIDVersionPostingsFormat.testGlobalVersions-seed#[504E9AE51F4C6630],
 state=TIMED_WAITING, group=TGRP-TestIDVersionPostingsFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4677)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2101)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2029)
   [junit4]   2>         at 
org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:440)
   [junit4]   2>         at 
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat.testGlobalVersions(TestIDVersionPostingsFormat.java:827)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>         at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestIDVersionPostingsFormat -Dtests.method=testGlobalVersions 
-Dtests.seed=504E9AE51F4C6630 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=es-GT -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7162s J1 | TestIDVersionPostingsFormat.testGlobalVersions 
<<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because 
suite timeout was reached.
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([504E9AE51F4C6630]:0)
   [junit4]   2> dic 14, 2017 4:42:54 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> dic 14, 2017 4:43:15 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at 
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat: 
   [junit4]   2>    1) Thread[id=88, 
name=TEST-TestIDVersionPostingsFormat.testGlobalVersions-seed#[504E9AE51F4C6630],
 state=TIMED_WAITING, group=TGRP-TestIDVersionPostingsFormat]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.doWait(IndexWriter.java:4677)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2101)
   [junit4]   2>         at 
org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:2029)
   [junit4]   2>         at 
org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:440)
   [junit4]   2>         at 
org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat.testGlobalVersions(TestIDVersionPostingsFormat.java:827)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>         at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
   [junit4]   2>         at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> dic 14, 2017 4:43:15 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=88, 
name=TEST-TestIDVersionPostingsFormat.testGlobalVersions-seed#[504E9AE51F4C6630],
 state=TIMED_WAITING, group=TGRP-TestIDVersionPostingsFormat]
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/sandbox/test/J1/temp/lucene.codecs.idversion.TestIDVersionPostingsFormat_504E9AE51F4C6630-001
   [junit4]   2> Dec 14, 2017 5:43:15 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=442, maxMBSortInHeap=5.480337413537019, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@7044fa52),
 locale=es-GT, timezone=Atlantic/Cape_Verde
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=211707680,total=347078656
   [junit4]   2> NOTE: All tests run in this JVM: [TestTermAutomatonQuery, 
TestLatLonPointDistanceSort, TestHalfFloatPoint, TestCoveringQuery, 
FuzzyLikeThisQueryTest, TestFieldCacheTermsFilter, TestIDVersionPostingsFormat]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestIDVersionPostingsFormat -Dtests.seed=504E9AE51F4C6630 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=es-GT -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestIDVersionPostingsFormat (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 
7200000 msec).
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([504E9AE51F4C6630]:0)
   [junit4] Completed [19/19 (1!)] on J1 in 7220.27s, 8 tests, 2 errors <<< 
FAILURES!

[...truncated 3056 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestHdfsCloudBackupRestore
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/init-core-data-001
   [junit4]   2> 1808458 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1808459 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1808485 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1808486 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1816421 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1826356 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1829587 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 1829977 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1834194 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log jetty-6.1.26
   [junit4]   2> 1834649 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_34765_hdfs____.yx261t/webapp
   [junit4]   2> 1839028 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34765
   [junit4]   2> 1850746 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1850881 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log jetty-6.1.26
   [junit4]   2> 1850963 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_59127_datanode____4t09h5/webapp
   [junit4]   2> 1853162 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:59127
   [junit4]   2> 1861642 WARN  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1861643 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log jetty-6.1.26
   [junit4]   2> 1862457 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_37500_datanode____.epqirs/webapp
   [junit4]   2> 1865257 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37500
   [junit4]   2> 1877534 ERROR (DataNode: 
[[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-001/hdfsBaseDir/data/data3/,
 
[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-001/hdfsBaseDir/data/data4/]]
  heartbeating to localhost/127.0.0.1:47531) [    ] 
o.a.h.h.s.d.DirectoryScanner 
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 
ms/sec. Assuming default value of 1000
   [junit4]   2> 1877534 ERROR (DataNode: 
[[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-001/hdfsBaseDir/data/data1/,
 
[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to localhost/127.0.0.1:47531) [    ] 
o.a.h.h.s.d.DirectoryScanner 
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 
ms/sec. Assuming default value of 1000
   [junit4]   2> 1880522 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x2ff04a55642918: from storage 
DS-4fbc0519-b82d-488c-a655-016a9b7ef904 node 
DatanodeRegistration(127.0.0.1:57989, 
datanodeUuid=c0198d88-ec9c-4ca3-9b0a-672ad6f1e758, infoPort=45050, 
infoSecurePort=0, ipcPort=54357, 
storageInfo=lv=-56;cid=testClusterID;nsid=1922486675;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 9 msecs
   [junit4]   2> 1880536 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x2ff04a532d0776: from storage 
DS-0b271aef-c4f6-4f11-a343-e47fef4821c1 node 
DatanodeRegistration(127.0.0.1:60726, 
datanodeUuid=a9c364a1-9098-4988-bf16-76c326f66510, infoPort=48188, 
infoSecurePort=0, ipcPort=38788, 
storageInfo=lv=-56;cid=testClusterID;nsid=1922486675;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1880690 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x2ff04a55642918: from storage 
DS-0039bc0b-1264-4f57-9057-7fcf975a6e52 node 
DatanodeRegistration(127.0.0.1:57989, 
datanodeUuid=c0198d88-ec9c-4ca3-9b0a-672ad6f1e758, infoPort=45050, 
infoSecurePort=0, ipcPort=54357, 
storageInfo=lv=-56;cid=testClusterID;nsid=1922486675;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1880690 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x2ff04a532d0776: from storage 
DS-e94fdb5f-d16b-4020-bf81-b3a146487897 node 
DatanodeRegistration(127.0.0.1:60726, 
datanodeUuid=a9c364a1-9098-4988-bf16-76c326f66510, infoPort=48188, 
infoSecurePort=0, ipcPort=38788, 
storageInfo=lv=-56;cid=testClusterID;nsid=1922486675;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1884726 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002
   [junit4]   2> 1884746 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1884750 INFO  (Thread-577) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1884750 INFO  (Thread-577) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 1884759 ERROR (Thread-577) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1884854 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:59944
   [junit4]   2> 1885002 INFO  (zkConnectionManagerCallback-456-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1885152 INFO  (jetty-launcher-453-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1885159 INFO  (jetty-launcher-453-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1885601 INFO  (jetty-launcher-453-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@63b02698{/solr,null,AVAILABLE}
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@3d2fc8ef{SSL,[ssl, 
http/1.1]}{127.0.0.1:36336}
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.e.j.s.Server Started @1906336ms
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=36336}
   [junit4]   2> 1885602 ERROR (jetty-launcher-453-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1885602 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-12-14T08:53:31.152Z
   [junit4]   2> 1885626 INFO  (jetty-launcher-453-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@54ecb9e8{/solr,null,AVAILABLE}
   [junit4]   2> 1885626 INFO  (jetty-launcher-453-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@1249c5ae{SSL,[ssl, 
http/1.1]}{127.0.0.1:47294}
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.e.j.s.Server Started @1906360ms
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=47294}
   [junit4]   2> 1885627 ERROR (jetty-launcher-453-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1885627 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-12-14T08:53:31.177Z
   [junit4]   2> 1885711 INFO  (zkConnectionManagerCallback-458-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1885712 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1885746 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1885847 INFO  (jetty-launcher-453-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59944/solr
   [junit4]   2> 1886234 INFO  (zkConnectionManagerCallback-460-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1886286 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1886346 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.c.SolrXmlConfig MBean server found: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747, but no JMX reporters were 
configured - adding default JMX reporter.
   [junit4]   2> 1886462 INFO  (jetty-launcher-453-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59944/solr
   [junit4]   2> 1886552 INFO  (zkConnectionManagerCallback-464-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1886733 INFO  (zkConnectionManagerCallback-468-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1886824 INFO  
(zkConnectionManagerCallback-470-thread-1-processing-n:127.0.0.1:36336_solr) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1887178 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1887178 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:36336_solr
   [junit4]   2> 1887197 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.Overseer Overseer 
(id=99171802172686341-127.0.0.1:36336_solr-n_0000000000) starting
   [junit4]   2> 1888608 INFO  
(zkConnectionManagerCallback-472-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1888652 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1888653 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:47294_solr
   [junit4]   2> 1889107 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:36336_solr
   [junit4]   2> 1889491 INFO  
(zkCallback-471-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 1889555 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 1890280 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.b.r.BackupRepositoryFactory Added backup 
repository with configuration params {type = repository,name = hdfs,class = 
org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = 
{name=hdfs, 
class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = 
{location=/backup,solr.hdfs.home=hdfs://localhost:47531/solr,solr.hdfs.confdir=}}
   [junit4]   2> 1890280 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.b.r.BackupRepositoryFactory Default 
configuration for backup repository is with configuration params {type = 
repository,name = hdfs,class = 
org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = 
{name=hdfs, 
class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = 
{location=/backup,solr.hdfs.home=hdfs://localhost:47531/solr,solr.hdfs.confdir=}}
   [junit4]   2> 1890924 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1892048 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1892049 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1892093 INFO  (jetty-launcher-453-thread-2) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002/node2/.
   [junit4]   2> 1892368 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.b.r.BackupRepositoryFactory Added backup 
repository with configuration params {type = repository,name = hdfs,class = 
org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = 
{name=hdfs, 
class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = 
{location=/backup,solr.hdfs.home=hdfs://localhost:47531/solr,solr.hdfs.confdir=}}
   [junit4]   2> 1892447 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.b.r.BackupRepositoryFactory Default 
configuration for backup repository is with configuration params {type = 
repository,name = hdfs,class = 
org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = 
{name=hdfs, 
class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = 
{location=/backup,solr.hdfs.home=hdfs://localhost:47531/solr,solr.hdfs.confdir=}}
   [junit4]   2> 1892949 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1893018 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1893061 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1893087 INFO  (jetty-launcher-453-thread-1) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002/node1/.
   [junit4]   2> 1894923 INFO  (zkConnectionManagerCallback-477-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1895038 INFO  (zkConnectionManagerCallback-481-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1895098 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1895147 INFO  
(SUITE-TestHdfsCloudBackupRestore-seed#[87653ED116AB22A0]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59944/solr ready
   [junit4]   2> 1895639 INFO  
(TEST-TestHdfsCloudBackupRestore.test-seed#[87653ED116AB22A0]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1896199 INFO  (qtp54371165-2226) [n:127.0.0.1:47294_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=4&router.field=shard_s&name=hdfsbackuprestore&nrtReplicas=2&action=CREATE&tlogReplicas=1&wt=javabin
 and sendToOCPQueue=true
   [junit4]   2> 1896227 INFO  
(OverseerThreadFactory-798-thread-1-processing-n:127.0.0.1:36336_solr) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
hdfsbackuprestore
   [junit4]   2> 1896864 WARN  
(OverseerThreadFactory-798-thread-1-processing-n:127.0.0.1:36336_solr) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.CreateCollectionCmd Specified number of 
replicas of 4 on collection hdfsbackuprestore is higher than the number of Solr 
instances currently live or live and part of your createNodeSet(2). It's 
unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 1897042 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897044 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36336/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897047 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_shard1_replica_t5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897049 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_shard1_replica_p7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36336/solr";,
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897051 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_shard2_replica_n9",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897054 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_shard2_replica_n11",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36336/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897056 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_shard2_replica_t13",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:47294/solr";,
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1897058 INFO  
(OverseerStateUpdate-99171802172686341-127.0.0.1:36336_solr-n_0000000000) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_shard2_replica_p15",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:36336/solr";,
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1898218 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node6&name=hdfsbackuprestore_shard1_replica_t5&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 1898219 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 1898219 INFO  (qtp54371165-2222) [n:127.0.0.1:47294_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node14&name=hdfsbackuprestore_shard2_replica_t13&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 1899650 INFO  
(zkCallback-471-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/hdfsbackuprestore/state.json] for collection 
[hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1899782 INFO  
(zkCallback-471-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/hdfsbackuprestore/state.json] for collection 
[hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1899848 INFO  (qtp54371165-2223) [n:127.0.0.1:47294_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node2&name=hdfsbackuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 1899875 INFO  (qtp54371165-2228) [n:127.0.0.1:47294_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=hdfsbackuprestore_shard2_replica_n9&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 1899994 INFO  (qtp857131042-2230) [n:127.0.0.1:36336_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node4&name=hdfsbackuprestore_shard1_replica_n3&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 1899994 INFO  (qtp857131042-2230) [n:127.0.0.1:36336_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 1900010 INFO  (qtp857131042-2231) [n:127.0.0.1:36336_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node16&name=hdfsbackuprestore_shard2_replica_p15&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 1900011 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node8&name=hdfsbackuprestore_shard1_replica_p7&action=CREATE&numShards=2&shard=shard1&wt=javabin
   [junit4]   2> 1900146 INFO  (qtp857131042-2237) [n:127.0.0.1:36336_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node12&name=hdfsbackuprestore_shard2_replica_n11&action=CREATE&numShards=2&shard=shard2&wt=javabin
   [junit4]   2> 1900271 INFO  
(zkCallback-471-thread-1-processing-n:127.0.0.1:47294_solr) 
[n:127.0.0.1:47294_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/hdfsbackuprestore/state.json] for collection 
[hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1900278 INFO  
(zkCallback-469-thread-1-processing-n:127.0.0.1:36336_solr) 
[n:127.0.0.1:36336_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/hdfsbackuprestore/state.json] for collection 
[hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1900685 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903688 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard1_replica_t5] Schema name=minimal
   [junit4]   2> 1903696 INFO  (qtp857131042-2237) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard2 r:core_node12 
x:hdfsbackuprestore_shard2_replica_n11] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903700 INFO  (qtp54371165-2223) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node2 
x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903704 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903708 INFO  (qtp54371165-2228) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard2 r:core_node10 
x:hdfsbackuprestore_shard2_replica_n9] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903712 INFO  (qtp54371165-2222) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard2 r:core_node14 
x:hdfsbackuprestore_shard2_replica_t13] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903715 INFO  (qtp857131042-2231) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard2 r:core_node16 
x:hdfsbackuprestore_shard2_replica_p15] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903718 INFO  (qtp857131042-2230) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node4 
x:hdfsbackuprestore_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 8.0.0
   [junit4]   2> 1903863 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.s.IndexSchema Loaded schema 
minimal/1.1 with uniqueid field id
   [junit4]   2> 1903863 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 
'hdfsbackuprestore_shard1_replica_t5' using configuration from collection 
hdfsbackuprestore, trusted=true
   [junit4]   2> 1903863 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring 
for 'solr.core.hdfsbackuprestore.shard1.replica_t5' (registry 
'solr.core.hdfsbackuprestore.shard1.replica_t5') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1903864 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 1903864 INFO  (qtp54371165-2229) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node6 
x:hdfsbackuprestore_shard1_replica_t5] o.a.s.c.SolrCore 
[[hdfsbackuprestore_shard1_replica_t5] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002/node2/hdfsbackuprestore_shard1_replica_t5],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002/node2/./hdfsbackuprestore_shard1_replica_t5/data/]
   [junit4]   2> 1903940 INFO  (qtp857131042-2237) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard2 r:core_node12 
x:hdfsbackuprestore_shard2_replica_n11] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard2_replica_n11] Schema name=minimal
   [junit4]   2> 1903944 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard1_replica_p7] Schema name=minimal
   [junit4]   2> 1903946 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.s.IndexSchema Loaded schema 
minimal/1.1 with uniqueid field id
   [junit4]   2> 1903946 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.c.CoreContainer Creating SolrCore 
'hdfsbackuprestore_shard1_replica_p7' using configuration from collection 
hdfsbackuprestore, trusted=true
   [junit4]   2> 1903964 INFO  (qtp857131042-2231) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard2 r:core_node16 
x:hdfsbackuprestore_shard2_replica_p15] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard2_replica_p15] Schema name=minimal
   [junit4]   2> 1903967 INFO  (qtp54371165-2223) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard1 r:core_node2 
x:hdfsbackuprestore_shard1_replica_n1] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1903969 INFO  (qtp54371165-2228) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard2 r:core_node10 
x:hdfsbackuprestore_shard2_replica_n9] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard2_replica_n9] Schema name=minimal
   [junit4]   2> 1903971 INFO  (qtp54371165-2222) [n:127.0.0.1:47294_solr 
c:hdfsbackuprestore s:shard2 r:core_node14 
x:hdfsbackuprestore_shard2_replica_t13] o.a.s.s.IndexSchema 
[hdfsbackuprestore_shard2_replica_t13] Schema name=minimal
   [junit4]   2> 1904005 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.m.r.SolrJmxReporter JMX monitoring 
for 'solr.core.hdfsbackuprestore.shard1.replica_p7' (registry 
'solr.core.hdfsbackuprestore.shard1.replica_p7') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4e59e747
   [junit4]   2> 1904005 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 1904005 INFO  (qtp857131042-2235) [n:127.0.0.1:36336_solr 
c:hdfsbackuprestore s:shard1 r:core_node8 
x:hdfsbackuprestore_shard1_replica_p7] o.a.s.c.SolrCore 
[[hdfsbackuprestore_shard1_replica_p7] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_87653ED116AB22A0-001/tempDir-002/node1/hdfsbackuprestore_shard1_replica_p7],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build

[...truncated too long message...]

]   2> 5535762 WARN  (LeaseRenewer:jenkins@localhost:47531) [    ] 
o.a.h.h.LeaseRenewer Failed to renew lease for 
[DFSClient_NONMAPREDUCE_-540540885_2347] for 3589 seconds.  Will retry shortly 
...
   [junit4]   2> java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 
to localhost:47531 failed on connection exception: java.net.ConnectException: 
Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2>        at 
sun.reflect.GeneratedConstructorAccessor292.newInstance(Unknown Source)
   [junit4]   2>        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
   [junit4]   2>        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1480)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1413)
   [junit4]   2>        at 
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
   [junit4]   2>        at com.sun.proxy.$Proxy59.renewLease(Unknown Source)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.renewLease(ClientNamenodeProtocolTranslatorPB.java:595)
   [junit4]   2>        at sun.reflect.GeneratedMethodAccessor76.invoke(Unknown 
Source)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
   [junit4]   2>        at 
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
   [junit4]   2>        at com.sun.proxy.$Proxy62.renewLease(Unknown Source)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:892)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.renew(LeaseRenewer.java:423)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:448)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:304)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused
   [junit4]   2>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native 
Method)
   [junit4]   2>        at 
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2>        at 
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:615)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:713)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:376)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client.getConnection(Client.java:1529)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
   [junit4]   2>        ... 16 more
   [junit4]   2> 5536775 WARN  (LeaseRenewer:jenkins@localhost:47531) [    ] 
o.a.h.h.LeaseRenewer Failed to renew lease for 
[DFSClient_NONMAPREDUCE_-540540885_2347] for 3590 seconds.  Will retry shortly 
...
   [junit4]   2> java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 
to localhost:47531 failed on connection exception: java.net.ConnectException: 
Connection refused; For more details see:  
http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2>        at 
sun.reflect.GeneratedConstructorAccessor292.newInstance(Unknown Source)
   [junit4]   2>        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
   [junit4]   2>        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1480)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1413)
   [junit4]   2>        at 
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
   [junit4]   2>        at com.sun.proxy.$Proxy59.renewLease(Unknown Source)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.renewLease(ClientNamenodeProtocolTranslatorPB.java:595)
   [junit4]   2>        at sun.reflect.GeneratedMethodAccessor76.invoke(Unknown 
Source)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
   [junit4]   2>        at 
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
   [junit4]   2>        at com.sun.proxy.$Proxy62.renewLease(Unknown Source)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:892)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.renew(LeaseRenewer.java:423)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:448)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:304)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused
   [junit4]   2>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native 
Method)
   [junit4]   2>        at 
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2>        at 
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531)
   [junit4]   2>        at 
org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:615)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:713)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:376)
   [junit4]   2>        at 
org.apache.hadoop.ipc.Client.getConnection(Client.java:1529)
   [junit4]   2>        at org.apache.hadoop.ipc.Client.call(Client.java:1452)
   [junit4]   2>        ... 16 more
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=60, maxMBSortInHeap=5.862360504732637, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3e6cd6b7),
 locale=sr, timezone=America/Montreal
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=206674968,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [TestConfigReload, 
TestPostingsSolrHighlighter, TestComponentsName, ResourceLoaderTest, 
StatelessScriptUpdateProcessorFactoryTest, TestEmbeddedSolrServerSchemaAPI, 
TestPerFieldSimilarityWithDefaultOverride, TestCloudPivotFacet, TestCSVLoader, 
TestMaxTokenLenTokenizer, TestCollectionsAPIViaSolrCloudCluster, 
TestLuceneIndexBackCompat, TestRandomFaceting, TestExportWriter, DateFieldTest, 
TestUninvertingReader, TestRandomRequestDistribution, TestStressVersions, 
SpellCheckCollatorTest, ClassificationUpdateProcessorIntegrationTest, 
IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, 
TestHdfsCloudBackupRestore, TestHighlightDedupGrouping, SolrIndexMetricsTest, 
AssignBackwardCompatibilityTest, TestReplicationHandlerBackup, 
QueryParsingTest, AutoAddReplicasIntegrationTest, ForceLeaderTest, 
SolrShardReporterTest, JsonLoaderTest, SpellCheckComponentTest, 
TestShardHandlerFactory, TestSmileRequest, DocumentBuilderTest, 
HttpSolrCallGetCoreTest, TestLegacyFieldCache, ExternalFileFieldSortTest, 
TestHashQParserPlugin, DocExpirationUpdateProcessorFactoryTest, 
CollectionsAPISolrJTest, TestPayloadScoreQParserPlugin, HdfsDirectoryTest, 
TestLeaderInitiatedRecoveryThread, DirectUpdateHandlerOptimizeTest, 
TestNestedDocsSort, MetricUtilsTest, TestNamedUpdateProcessors, 
TestFileDictionaryLookup, TestDistributedSearch, 
OverseerCollectionConfigSetProcessorTest, TestSolrXml, 
SegmentsInfoRequestHandlerTest, PKIAuthenticationIntegrationTest, 
TestSQLHandler, AnalyticsMergeStrategyTest, TestMinMaxOnMultiValuedField, 
TestRuleBasedAuthorizationPlugin, TestPHPSerializedResponseWriter, 
DeleteReplicaTest, DateMathParserTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DateMathParserTest 
-Dtests.seed=87653ED116AB22A0 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=sr -Dtests.timezone=America/Montreal -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | DateMathParserTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The test or suite 
printed 11184 bytes to stdout and stderr, even though the limit was set to 8192 
bytes. Increase the limit with @Limit, ignore it completely with 
@SuppressSysoutChecks or run with -Dtests.verbose=true
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([87653ED116AB22A0]:0)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [678/753 (5!)] on J0 in 1.73s, 11 tests, 1 failure <<< 
FAILURES!

[...truncated 47610 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to