Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1449/

7 tests failed.
FAILED:  org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates

Error Message:
_1_l.fnm in dir=RAMDirectory@e8d59da 
lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5cdd87eb

Stack Trace:
java.nio.file.NoSuchFileException: _1_l.fnm in dir=RAMDirectory@e8d59da 
lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5cdd87eb
        at 
__randomizedtesting.SeedInfo.seed([7D3E6928A520CDD7:51BB7234700E235]:0)
        at 
org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:750)
        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
        at 
org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
        at 
org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates(TestMixedDocValuesUpdates.java:374)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RestartWhileUpdatingTest.test

Error Message:
shard1 is not consistent.  Got 1007 from 
https://127.0.0.1:46010/y/hi/collection1_shard1_replica_n21 (previous client) 
and got 1248 from https://127.0.0.1:44643/y/hi/collection1_shard1_replica_n23

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 1007 from 
https://127.0.0.1:46010/y/hi/collection1_shard1_replica_n21 (previous client) 
and got 1248 from https://127.0.0.1:44643/y/hi/collection1_shard1_replica_n23
        at 
__randomizedtesting.SeedInfo.seed([6188612105A09390:E9DC5EFBAB5CFE68]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1328)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1307)
        at 
org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:155)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest

Error Message:
1 thread leaked from SUITE scope at 
org.apache.solr.cloud.RestartWhileUpdatingTest:     1) Thread[id=12319, 
name=searcherExecutor-2251-thread-1, state=WAITING, 
group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.cloud.RestartWhileUpdatingTest: 
   1) Thread[id=12319, name=searcherExecutor-2251-thread-1, state=WAITING, 
group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([6188612105A09390]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=12319, name=searcherExecutor-2251-thread-1, state=WAITING, 
group=TGRP-RestartWhileUpdatingTest]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=12319, name=searcherExecutor-2251-thread-1, state=WAITING, 
group=TGRP-RestartWhileUpdatingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([6188612105A09390]:0)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners

Error Message:
foo events: [CapturedEvent{timestamp=452002502401546, stage=STARTED, 
actionName='null', event={   "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",   
"source":"node_added_trigger",   "eventTime":452000430428996,   
"eventType":"NODEADDED",   "properties":{     "eventTimes":[452000430428996],   
  "_enqueue_time_":452002483050696,     "nodeNames":["127.0.0.1:10013_solr"]}}, 
context={}, config={   "trigger":"node_added_trigger",   "stage":[     
"STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "afterAction":[     
"test",     "test1"],   
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
   "beforeAction":"test"}, message='null'}, 
CapturedEvent{timestamp=452002509401596, stage=BEFORE_ACTION, 
actionName='test', event={   "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",   
"source":"node_added_trigger",   "eventTime":452000430428996,   
"eventType":"NODEADDED",   "properties":{     "eventTimes":[452000430428996],   
  "_enqueue_time_":452002483050696,     "nodeNames":["127.0.0.1:10013_solr"]}}, 
context={properties.BEFORE_ACTION=[test], source=node_added_trigger}, config={  
 "trigger":"node_added_trigger",   "stage":[     "STARTED",     "ABORTED",     
"SUCCEEDED",     "FAILED"],   "afterAction":[     "test",     "test1"],   
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
   "beforeAction":"test"}, message='null'}, 
CapturedEvent{timestamp=452002511149446, stage=AFTER_ACTION, actionName='test', 
event={   "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",   
"source":"node_added_trigger",   "eventTime":452000430428996,   
"eventType":"NODEADDED",   "properties":{     "eventTimes":[452000430428996],   
  "_enqueue_time_":452002483050696,     "nodeNames":["127.0.0.1:10013_solr"]}}, 
context={properties.BEFORE_ACTION=[test], source=node_added_trigger, 
properties.AFTER_ACTION=[test]}, config={   "trigger":"node_added_trigger",   
"stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   
"afterAction":[     "test",     "test1"],   
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
   "beforeAction":"test"}, message='null'}] expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: foo events: [CapturedEvent{timestamp=452002502401546, 
stage=STARTED, actionName='null', event={
  "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",
  "source":"node_added_trigger",
  "eventTime":452000430428996,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[452000430428996],
    "_enqueue_time_":452002483050696,
    "nodeNames":["127.0.0.1:10013_solr"]}}, context={}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}, 
CapturedEvent{timestamp=452002509401596, stage=BEFORE_ACTION, 
actionName='test', event={
  "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",
  "source":"node_added_trigger",
  "eventTime":452000430428996,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[452000430428996],
    "_enqueue_time_":452002483050696,
    "nodeNames":["127.0.0.1:10013_solr"]}}, 
context={properties.BEFORE_ACTION=[test], source=node_added_trigger}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}, 
CapturedEvent{timestamp=452002511149446, stage=AFTER_ACTION, actionName='test', 
event={
  "id":"19b178d181344Tdlh6rgdzkoyndpp6uu9gvesbb",
  "source":"node_added_trigger",
  "eventTime":452000430428996,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[452000430428996],
    "_enqueue_time_":452002483050696,
    "nodeNames":["127.0.0.1:10013_solr"]}}, 
context={properties.BEFORE_ACTION=[test], source=node_added_trigger, 
properties.AFTER_ACTION=[test]}, config={
  "trigger":"node_added_trigger",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "afterAction":[
    "test",
    "test1"],
  
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":"test"}, message='null'}] expected:<4> but was:<3>
        at 
__randomizedtesting.SeedInfo.seed([6188612105A09390:8C48F631B8CC0E49]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:1009)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig

Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{   
"cluster-preferences":[{"maximize":"freedisk"}],   
"triggers":{".auto_add_replicas":{       "name":".auto_add_replicas",       
"event":"nodeLost",       "waitFor":30,       "actions":[         {           
"name":"auto_add_replicas_plan",           
"class":"solr.AutoAddReplicasPlanAction"},         {           
"name":"execute_plan",           "class":"solr.ExecutePlanAction"}],       
"enabled":true}},   "listeners":{".auto_add_replicas.system":{       
"trigger":".auto_add_replicas",       "afterAction":[],       "stage":[         
"STARTED",         "ABORTED",         "SUCCEEDED",         "FAILED",         
"BEFORE_ACTION",         "AFTER_ACTION",         "IGNORED"],       
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",       
"beforeAction":[]}},   "properties":{}}> but was: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{   
"cluster-preferences":[{"maximize":"freedisk"}],   
"triggers":{".auto_add_replicas":{       "name":".auto_add_replicas",       
"event":"nodeLost",       "waitFor":30,       "actions":[         {           
"name":"auto_add_replicas_plan",           
"class":"solr.AutoAddReplicasPlanAction"},         {           
"name":"execute_plan",           "class":"solr.ExecutePlanAction"}],       
"enabled":true}},   "listeners":{".auto_add_replicas.system":{       
"trigger":".auto_add_replicas",       "afterAction":[],       "stage":[         
"STARTED",         "ABORTED",         "SUCCEEDED",         "FAILED",         
"BEFORE_ACTION",         "AFTER_ACTION",         "IGNORED"],       
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",       
"beforeAction":[]}},   "properties":{}}>

Stack Trace:
java.lang.AssertionError: expected: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
  "cluster-preferences":[{"maximize":"freedisk"}],
  "triggers":{".auto_add_replicas":{
      "name":".auto_add_replicas",
      "event":"nodeLost",
      "waitFor":30,
      "actions":[
        {
          "name":"auto_add_replicas_plan",
          "class":"solr.AutoAddReplicasPlanAction"},
        {
          "name":"execute_plan",
          "class":"solr.ExecutePlanAction"}],
      "enabled":true}},
  "listeners":{".auto_add_replicas.system":{
      "trigger":".auto_add_replicas",
      "afterAction":[],
      "stage":[
        "STARTED",
        "ABORTED",
        "SUCCEEDED",
        "FAILED",
        "BEFORE_ACTION",
        "AFTER_ACTION",
        "IGNORED"],
      "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
      "beforeAction":[]}},
  "properties":{}}> but was: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
  "cluster-preferences":[{"maximize":"freedisk"}],
  "triggers":{".auto_add_replicas":{
      "name":".auto_add_replicas",
      "event":"nodeLost",
      "waitFor":30,
      "actions":[
        {
          "name":"auto_add_replicas_plan",
          "class":"solr.AutoAddReplicasPlanAction"},
        {
          "name":"execute_plan",
          "class":"solr.ExecutePlanAction"}],
      "enabled":true}},
  "listeners":{".auto_add_replicas.system":{
      "trigger":".auto_add_replicas",
      "afterAction":[],
      "stage":[
        "STARTED",
        "ABORTED",
        "SUCCEEDED",
        "FAILED",
        "BEFORE_ACTION",
        "AFTER_ACTION",
        "IGNORED"],
      "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
      "beforeAction":[]}},
  "properties":{}}>
        at 
__randomizedtesting.SeedInfo.seed([6188612105A09390:5E00628912F66377]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1516002348100,generation=2,filelist=[_6u8.fdt, 
_6u8.fdx, _6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, 
_7tf.fdx, _7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, 
_9cs.fdx, _9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, 
_d4x.fdx, _d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, 
_e66.cfs, _e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _ea2.cfe, _ea2.cfs, _ea2.si, 
_ea3.cfe, _ea3.cfs, _ea3.si, _ea4.cfe, _ea4.cfs, _ea4.si, _ea5.cfe, _ea5.cfs, 
_ea5.si, _ea6.cfe, _ea6.cfs, _ea6.si, _ea7.cfe, _ea7.cfs, _ea7.si, _ea8.cfe, 
_ea8.cfs, _ea8.si, _ea9.cfe, _ea9.cfs, _ea9.si, _eaa.cfe, _eaa.cfs, _eaa.si, 
_eab.cfe, _eab.cfs, _eab.si, _ead.cfe, _ead.cfs, _ead.si, segments_2]}]> but 
was:<[{indexVersion=1516002348100,generation=2,filelist=[_6u8.fdt, _6u8.fdx, 
_6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, _7tf.fdx, 
_7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, _9cs.fdx, 
_9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, _d4x.fdx, 
_d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, _e66.cfs, 
_e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _ea2.cfe, _ea2.cfs, _ea2.si, _ea3.cfe, 
_ea3.cfs, _ea3.si, _ea4.cfe, _ea4.cfs, _ea4.si, _ea5.cfe, _ea5.cfs, _ea5.si, 
_ea6.cfe, _ea6.cfs, _ea6.si, _ea7.cfe, _ea7.cfs, _ea7.si, _ea8.cfe, _ea8.cfs, 
_ea8.si, _ea9.cfe, _ea9.cfs, _ea9.si, _eaa.cfe, _eaa.cfs, _eaa.si, _eab.cfe, 
_eab.cfs, _eab.si, _ead.cfe, _ead.cfs, _ead.si, segments_2]}, 
{indexVersion=1516002348100,generation=3,filelist=[_6u8.fdt, _6u8.fdx, 
_6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, _7tf.fdx, 
_7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, _9cs.fdx, 
_9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, _d4x.fdx, 
_d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, _e66.cfs, 
_e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _eac.cfe, _eac.cfs, _eac.si, _ead.cfe, 
_ead.cfs, _ead.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: 
expected:<[{indexVersion=1516002348100,generation=2,filelist=[_6u8.fdt, 
_6u8.fdx, _6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, 
_7tf.fdx, _7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, 
_9cs.fdx, _9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, 
_d4x.fdx, _d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, 
_e66.cfs, _e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _ea2.cfe, _ea2.cfs, _ea2.si, 
_ea3.cfe, _ea3.cfs, _ea3.si, _ea4.cfe, _ea4.cfs, _ea4.si, _ea5.cfe, _ea5.cfs, 
_ea5.si, _ea6.cfe, _ea6.cfs, _ea6.si, _ea7.cfe, _ea7.cfs, _ea7.si, _ea8.cfe, 
_ea8.cfs, _ea8.si, _ea9.cfe, _ea9.cfs, _ea9.si, _eaa.cfe, _eaa.cfs, _eaa.si, 
_eab.cfe, _eab.cfs, _eab.si, _ead.cfe, _ead.cfs, _ead.si, segments_2]}]> but 
was:<[{indexVersion=1516002348100,generation=2,filelist=[_6u8.fdt, _6u8.fdx, 
_6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, _7tf.fdx, 
_7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, _9cs.fdx, 
_9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, _d4x.fdx, 
_d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, _e66.cfs, 
_e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _ea2.cfe, _ea2.cfs, _ea2.si, _ea3.cfe, 
_ea3.cfs, _ea3.si, _ea4.cfe, _ea4.cfs, _ea4.si, _ea5.cfe, _ea5.cfs, _ea5.si, 
_ea6.cfe, _ea6.cfs, _ea6.si, _ea7.cfe, _ea7.cfs, _ea7.si, _ea8.cfe, _ea8.cfs, 
_ea8.si, _ea9.cfe, _ea9.cfs, _ea9.si, _eaa.cfe, _eaa.cfs, _eaa.si, _eab.cfe, 
_eab.cfs, _eab.si, _ead.cfe, _ead.cfs, _ead.si, segments_2]}, 
{indexVersion=1516002348100,generation=3,filelist=[_6u8.fdt, _6u8.fdx, 
_6u8.fnm, _6u8.nvd, _6u8.nvm, _6u8.si, _6u8_Memory_0.ram, _7tf.fdt, _7tf.fdx, 
_7tf.fnm, _7tf.nvd, _7tf.nvm, _7tf.si, _7tf_Memory_0.ram, _9cs.fdt, _9cs.fdx, 
_9cs.fnm, _9cs.nvd, _9cs.nvm, _9cs.si, _9cs_Memory_0.ram, _d4x.fdt, _d4x.fdx, 
_d4x.fnm, _d4x.nvd, _d4x.nvm, _d4x.si, _d4x_Memory_0.ram, _e66.cfe, _e66.cfs, 
_e66.si, _e9t.cfe, _e9t.cfs, _e9t.si, _eac.cfe, _eac.cfs, _eac.si, _ead.cfe, 
_ead.cfs, _ead.si, segments_3]}]>
        at 
__randomizedtesting.SeedInfo.seed([6188612105A09390:445F7A1175E89D93]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 456 lines...]
   [junit4] Suite: org.apache.lucene.index.TestMixedDocValuesUpdates
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestMixedDocValuesUpdates -Dtests.method=testTonsOfUpdates 
-Dtests.seed=7D3E6928A520CDD7 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=es-ES -Dtests.timezone=Etc/UTC -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   23.4s J2 | TestMixedDocValuesUpdates.testTonsOfUpdates <<<
   [junit4]    > Throwable #1: java.nio.file.NoSuchFileException: _1_l.fnm in 
dir=RAMDirectory@e8d59da 
lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5cdd87eb
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([7D3E6928A520CDD7:51BB7234700E235]:0)
   [junit4]    >        at 
org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:750)
   [junit4]    >        at 
org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
   [junit4]    >        at 
org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
   [junit4]    >        at 
org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
   [junit4]    >        at 
org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
   [junit4]    >        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:705)
   [junit4]    >        at 
org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:332)
   [junit4]    >        at 
org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:740)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5104)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5094)
   [junit4]    >        at 
org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1889)
   [junit4]    >        at 
org.apache.lucene.index.TestMixedDocValuesUpdates.testTonsOfUpdates(TestMixedDocValuesUpdates.java:374)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestMixedDocValuesUpdates_7D3E6928A520CDD7-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3299b0f7),
 locale=es-ES, timezone=Etc/UTC
   [junit4]   2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=118090952,total=368574464
   [junit4]   2> NOTE: All tests run in this JVM: [TestExceedMaxTermLength, 
TestConsistentFieldNumbers, TestCompiledAutomaton, TestAssertions, 
TestTermScorer, TestBKD, TestStopFilter, TestIntroSelector, 
TestDoubleValuesSource, TestRecyclingIntBlockAllocator, TestRoaringDocIdSet, 
Test2BPostingsBytes, TestTermRangeQuery, TestShardSearching, TestIndexCommit, 
TestLucene60PointsFormat, TestSparseFixedBitDocIdSet, TestNRTThreads, 
TestReqExclBulkScorer, TestSpanSearchEquivalence, TestWANDScorer, 
TestBlockPostingsFormat, TestBooleanScorer, TestMixedDocValuesUpdates]
   [junit4] Completed [65/476 (1!)] on J2 in 31.68s, 4 tests, 1 error <<< 
FAILURES!

[...truncated 12313 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 1013372 INFO  
(SUITE-TestTriggerIntegration-seed#[6188612105A09390]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_6188612105A09390-001/init-core-data-001
   [junit4]   2> 1013385 WARN  
(SUITE-TestTriggerIntegration-seed#[6188612105A09390]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
   [junit4]   2> 1013385 INFO  
(SUITE-TestTriggerIntegration-seed#[6188612105A09390]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1013386 INFO  
(SUITE-TestTriggerIntegration-seed#[6188612105A09390]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1013425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1013425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 1013425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: []
   [junit4]   2> 1013425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1013425 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1013427 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1013427 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 1013428 DEBUG (ScheduledTrigger-709-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013443 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testCooldown
   [junit4]   2> 1013443 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 1013443 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1013444 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1013539 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1013539 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1013539 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1013540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1013540 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013559 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1013559 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1013560 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013591 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013612 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013632 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013656 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013663 DEBUG (simCloudManagerPool-711-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1013664 DEBUG (simCloudManagerPool-711-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 1013688 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1013690 DEBUG (simCloudManagerPool-711-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 1013690 DEBUG (simCloudManagerPool-711-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1013690 DEBUG (simCloudManagerPool-711-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 1013690 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1013690 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1013691 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1013691 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013691 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1013691 DEBUG (simCloudManagerPool-711-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1013691 DEBUG (simCloudManagerPool-711-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 1013691 DEBUG (simCloudManagerPool-711-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 1013691 DEBUG (simCloudManagerPool-711-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1013691 DEBUG (simCloudManagerPool-711-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 1013692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1013692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1013709 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1013712 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013712 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1013712 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 
451836432041346
   [junit4]   2> 1013729 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1013732 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013732 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1013732 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10002_solr] added at times 
[451836432041346], now=451837455684596
   [junit4]   2> 1013732 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af15e074582Tdlh6rgdzkoyndpp6uu9gves9g",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451836432041346,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451836432041346],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 1013754 DEBUG (AutoscalingActionExecutor-713-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"19af15e074582Tdlh6rgdzkoyndpp6uu9gves9g",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451836432041346,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451836432041346],
   [junit4]   2>     "_enqueue_time_":451837471327596,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 1013756 DEBUG (AutoscalingActionExecutor-713-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 65 ms for event 
id=19af15e074582Tdlh6rgdzkoyndpp6uu9gves9g
   [junit4]   2> 1013769 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1013775 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013775 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1013790 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013795 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013795 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013795 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 
451840609946996
   [junit4]   2> 1013810 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013815 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013815 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013815 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10003_solr] added at times 
[451840609946996], now=451841624097596
   [junit4]   2> 1013816 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"19af2570d0d74Tdlh6rgdzkoyndpp6uu9gves9j",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451840609946996,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451840609946996],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 1013830 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013836 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013836 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013836 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10003_solr] added at times 
[451840609946996], now=451842666311246
   [junit4]   2> 1013836 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"19af2570d0d74Tdlh6rgdzkoyndpp6uu9gves9l",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451840609946996,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451840609946996],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 1013850 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013857 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013857 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013857 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10003_solr] added at times 
[451840609946996], now=451843699502796
   [junit4]   2> 1013857 DEBUG (ScheduledTrigger-712-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af2570d0d74Tdlh6rgdzkoyndpp6uu9gves9n",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451840609946996,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451840609946996],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 1013858 DEBUG (AutoscalingActionExecutor-713-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"19af2570d0d74Tdlh6rgdzkoyndpp6uu9gves9n",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":451840609946996,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451840609946996],
   [junit4]   2>     "_enqueue_time_":451843706473946,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 1013861 DEBUG (AutoscalingActionExecutor-713-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 196 ms for event 
id=19af2570d0d74Tdlh6rgdzkoyndpp6uu9gves9n
   [junit4]   2> 1013870 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013879 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013879 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013891 DEBUG (ScheduledTrigger-712-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testCooldown
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:                4
   [junit4]   2> 1013898 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:       4
   [junit4]   2> 1013899 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:                0
   [junit4]   2> 1013899 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013899 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013899 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:       []
   [junit4]   2> 1013899 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:     0
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:     0
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:            0
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - autoscaling        2
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - update             6
   [junit4]   2> 1013900 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1013901 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_cooldown_trigger
   [junit4]   2> 1013901 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - IGNORED         2
   [junit4]   2> 1013901 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - STARTED         2
   [junit4]   2> 1013901 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[6188612105A09390]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - SUCCEEDED       2
   [junit4]   2> 1013911 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013927 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1013928 DEBUG (ScheduledTrigger-712-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1013931 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[6188612105A09390]) [   
 ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 1013931 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[6188612105A09390]) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 1013931 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 1013931 DEBUG (ScheduledTrigger-712-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1013932 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1013932 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1013932 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 1013942 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[6188612105A09390]) [   
 ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1013942 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1013943 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[6188612105A09390]) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1013957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1013957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 6
   [junit4]   2> 1013957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1013957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 1013957 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1013978 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1013978 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1013979 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014013 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014049 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014089 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014102 DEBUG (simCloudManagerPool-714-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1014102 DEBUG (simCloudManagerPool-714-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 1014104 DEBUG (simCloudManagerPool-714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1014104 DEBUG (simCloudManagerPool-714-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1014104 DEBUG (simCloudManagerPool-714-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014104 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 1014104 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1014105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 1014115 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014115 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014115 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 1014116 DEBUG (simCloudManagerPool-714-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014116 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 1014117 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1014117 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 1014117 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014117 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014117 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
451856690014246
   [junit4]   2> 1014117 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[451856690014246], now=451856691269046
   [junit4]   2> 1014117 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af6157f6826Tdlh6rgdzkoyndpp6uu9gves9q",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":451856690014246,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451856690014246],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014117 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"19af6157f6826Tdlh6rgdzkoyndpp6uu9gves9q",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":451856690014246,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451856690014246],
   [junit4]   2>     "_enqueue_time_":451856697838896,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014117 INFO  (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 1014130 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 655 ms for event 
id=19af6157f6826Tdlh6rgdzkoyndpp6uu9gves9q
   [junit4]   2> 1014159 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1014160 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014160 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014160 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
451858851432046
   [junit4]   2> 1014160 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[451858851432046], now=451858852877246
   [junit4]   2> 1014160 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"19af69654066eTdlh6rgdzkoyndpp6uu9gves9t",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":451858851432046,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451858851432046],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014178 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014178 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014179 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1014198 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014198 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014198 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014198 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014199 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[451858851432046], now=451860777721096
   [junit4]   2> 1014199 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"19af69654066eTdlh6rgdzkoyndpp6uu9gves9v",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":451858851432046,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451858851432046],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014207 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1014219 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014219 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014228 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1014247 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014248 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1014248 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[451858851432046], now=451863230014446
   [junit4]   2> 1014248 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af69654066eTdlh6rgdzkoyndpp6uu9gves9x",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":451858851432046,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451858851432046],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014248 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"19af69654066eTdlh6rgdzkoyndpp6uu9gves9x",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":451858851432046,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451858851432046],
   [junit4]   2>     "_enqueue_time_":451863237594046,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014248 INFO  (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration last action at 451856726100596 time = 
451863268101096
   [junit4]   2> 1014248 INFO  (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 1014250 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 77 ms for event 
id=19af69654066eTdlh6rgdzkoyndpp6uu9gves9x
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
9
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014264 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014265 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014265 DEBUG (simCloudManagerPool-714-thread-9) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 1014265 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1014265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 1014265 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 3
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
10
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014266 DEBUG (simCloudManagerPool-714-thread-10) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 1014266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
10
   [junit4]   2> 1014267 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1014267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1014267 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 3
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014268 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 2
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10004_solr]
   [junit4]   2> 1014301 DEBUG (ScheduledTrigger-715-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af83b854f16Tdlh6rgdzkoyndpp6uu9gvesa0",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":451865917869846,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451865917869846],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014302 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 2
   [junit4]   2> 1014302 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 1014302 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10004_solr]
   [junit4]   2> 1014302 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"19af83d036e86Tdlh6rgdzkoyndpp6uu9gvesa1",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":451865942912646,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451865942912646],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014393 DEBUG (ScheduledTrigger-715-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: 
[127.0.0.1:10004_solr] is not ready, will try later
   [junit4]   2> 1014410 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"19af83b854f16Tdlh6rgdzkoyndpp6uu9gvesa0",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":451865917869846,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[451865917869846],
   [junit4]   2>     "_enqueue_time_":451865925492996,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 1014410 INFO  (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger1
   [junit4]   2> 1014413 DEBUG (AutoscalingActionExecutor-716-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 121 ms for event 
id=19af83b854f16Tdlh6rgdzkoyndpp6uu9gvesa0
   [junit4]   2> 1014430 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1014430 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1014430 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1014430 DEBUG (ScheduledTrigger-715-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 101

[...truncated too long message...]

[junit4]   2> 5876890 INFO  (indexFetcher-10216-thread-1) [    x:collection1] 
o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 5876891 ERROR (indexFetcher-10210-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Index fetch failed 
:org.apache.solr.common.SolrException: SolrCoreState already closed.
   [junit4]   2>        at 
org.apache.solr.update.DefaultSolrCoreState.lock(DefaultSolrCoreState.java:186)
   [junit4]   2>        at 
org.apache.solr.update.DefaultSolrCoreState.newIndexWriter(DefaultSolrCoreState.java:227)
   [junit4]   2>        at 
org.apache.solr.update.DirectUpdateHandler2.newIndexWriter(DirectUpdateHandler2.java:766)
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:646)
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:345)
   [junit4]   2>        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
   [junit4]   2>        at 
org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 5876891 ERROR (indexFetcher-10216-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Index fetch failed 
:org.apache.solr.common.SolrException: Index fetch failed : 
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:672)
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:345)
   [junit4]   2>        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
   [junit4]   2>        at 
org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$12(ReplicationHandler.java:1159)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:905)
   [junit4]   2>        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:649)
   [junit4]   2>        ... 10 more
   [junit4]   2> 
   [junit4]   2> 5888990 ERROR (coreCloseExecutor-10220-thread-1) [    
x:collection1] o.a.s.c.CachingDirectoryFactory Timeout waiting for all 
directory ref counts to be released - gave up waiting on 
CachedDir<<refCount=1;path=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6188612105A09390-001/solr-instance-032/./collection1/data/index.20180115044551281;done=false>>
   [junit4]   2> 5888990 ERROR (coreCloseExecutor-10220-thread-1) [    
x:collection1] o.a.s.c.CachingDirectoryFactory Error closing 
directory:org.apache.solr.common.SolrException: Timeout waiting for all 
directory ref counts to be released - gave up waiting on 
CachedDir<<refCount=1;path=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6188612105A09390-001/solr-instance-032/./collection1/data/index.20180115044551281;done=false>>
   [junit4]   2>        at 
org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:178)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.close(SolrCore.java:1633)
   [junit4]   2>        at 
org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 5888990 ERROR (coreCloseExecutor-10220-thread-1) [    
x:collection1] o.a.s.c.SolrCore java.lang.AssertionError: 1
   [junit4]   2>        at 
org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.close(SolrCore.java:1633)
   [junit4]   2>        at 
org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 5888990 ERROR (coreCloseExecutor-10220-thread-1) [    
x:collection1] o.a.s.c.SolrCores Error shutting down 
core:java.lang.AssertionError: 1
   [junit4]   2>        at 
org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.close(SolrCore.java:1633)
   [junit4]   2>        at 
org.apache.solr.core.SolrCores.lambda$close$0(SolrCores.java:141)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 5888991 INFO  
(TEST-TestReplicationHandler.doTestIndexAndConfigReplication-seed#[6188612105A09390])
 [    ] o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@6c33eb26{/solr,null,UNAVAILABLE}
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6188612105A09390-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{name=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory), 
newname=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=1194, 
maxMBSortInHeap=6.955543651530648, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1370b56d),
 locale=es-AR, timezone=America/Argentina/Tucuman
   [junit4]   2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=384106616,total=529006592
   [junit4]   2> NOTE: All tests run in this JVM: [CollectionStateFormat2Test, 
TestUseDocValuesAsStored, CloudMLTQParserTest, TestUtils, TestPullReplica, 
TestSolrXml, TestDistributedSearch, BasicZkTest, TestBinaryField, 
TestWordDelimiterFilterFactory, TestFastOutputStream, TestCrossCoreJoin, 
TestFieldCacheReopen, VersionInfoTest, TestTriggerIntegration, 
TestComputePlanAction, TestSchemaSimilarityResource, DateMathParserTest, 
TestLeaderInitiatedRecoveryThread, TestSolrConfigHandlerConcurrent, 
DocumentBuilderTest, TestSweetSpotSimilarityFactory, 
TestPostingsSolrHighlighter, TestConfig, ForceLeaderTest, TermsComponentTest, 
TestSolr4Spatial2, CustomHighlightComponentTest, SubstringBytesRefFilterTest, 
ZkControllerTest, ExplicitHLLTest, NoCacheHeaderTest, 
TestHdfsCloudBackupRestore, RequestLoggingTest, SSLMigrationTest, 
JavabinLoaderTest, RAMDirectoryFactoryTest, TestMergePolicyConfig, 
TestSegmentSorting, RecoveryAfterSoftCommitTest, ChangedSchemaMergeTest, 
CursorPagingTest, TestComponentsName, TestStressLiveNodes, 
CoreAdminRequestStatusTest, DataDrivenBlockJoinTest, TestObjectReleaseTracker, 
HdfsCollectionsAPIDistributedZkTest, TestCloudPivotFacet, 
DirectUpdateHandlerOptimizeTest, TestInPlaceUpdatesDistrib, MultiTermTest, 
TestSolrCloudWithHadoopAuthPlugin, TestSolrDeletionPolicy1, NodeMutatorTest, 
AnalysisAfterCoreReloadTest, FacetPivotSmallTest, TestJoin, JSONWriterTest, 
TestManagedSchema, TestSQLHandler, TestBadConfig, TestGenericDistributedQueue, 
CdcrBootstrapTest, QueryParsingTest, MergeStrategyTest, 
TestSubQueryTransformerCrossCore, ResponseLogComponentTest, 
TestStandardQParsers, TestReplicationHandlerBackup, 
UpdateRequestProcessorFactoryTest, DeleteInactiveReplicaTest, TestStreamBody, 
AddBlockUpdateTest, TestUniqueKeyFieldResource, V2StandaloneTest, 
HdfsBasicDistributedZkTest, AnalyticsQueryTest, 
ClassificationUpdateProcessorTest, GraphQueryTest, TestReplicationHandler]
   [junit4] Completed [721/767 (4!)] on J0 in 336.27s, 16 tests, 1 failure, 1 
skipped <<< FAILURES!

[...truncated 48150 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to