Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2270/

2 tests failed.
FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<5> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<0>
        at 
__randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:E20937E07A1EA8]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:274)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10010_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10010_solr 
should have been deleted
        at 
__randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:75A425C654072C40]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertFalse(Assert.java:68)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12638 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_6D1EADCA5A32E1AF-001/init-core-data-001
   [junit4]   2> 1648503 WARN  
(SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=41 numCloses=41
   [junit4]   2> 1648504 INFO  
(SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1648525 INFO  
(SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1648526 INFO  
(SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1648527 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1648527 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 1648528 DEBUG (ScheduledTrigger-4611-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648542 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 1648542 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 1
   [junit4]   2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 1648542 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1648542 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1648543 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1648561 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1648565 DEBUG (ScheduledTrigger-4614-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648585 DEBUG (ScheduledTrigger-4614-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648605 DEBUG (ScheduledTrigger-4614-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648626 DEBUG (ScheduledTrigger-4614-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648646 DEBUG (ScheduledTrigger-4614-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648662 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1
   [junit4]   2> 1648662 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1648663 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1648663 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10008_solr
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1648669 DEBUG (ScheduledTrigger-4617-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648670 DEBUG (ScheduledTrigger-4617-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr
   [junit4]   2> 1648690 DEBUG (ScheduledTrigger-4617-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648710 DEBUG (ScheduledTrigger-4617-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648739 DEBUG (ScheduledTrigger-4617-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648772 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS
   [junit4]   2> 1648776 DEBUG (ScheduledTrigger-4617-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1648793 DEBUG (simCloudManagerPool-4616-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1648793 DEBUG (simCloudManagerPool-4616-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 1648829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1648829 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 1648829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 1648842 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1
   [junit4]   2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1648842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10010_solr at time 
1144705664944789
   [junit4]   2> 1648842 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 1648842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1648845 DEBUG (ScheduledTrigger-4617-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 3
   [junit4]   2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10010_solr] added at times 
[1144705664944789], now=1144707034696639
   [junit4]   2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"4111a8e880a95T38bxngjjsw1zhb3lymth2blvb",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144705664944789,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144705664944789],
   [junit4]   2>     "nodeNames":["127.0.0.1:10010_solr"]}}
   [junit4]   2> 1648870 DEBUG (ScheduledTrigger-4617-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_lost_trigger, node_added_trigger]
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:         3
   [junit4]   2> 1648950 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:        3
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:         1
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##             127.0.0.1:10008_solr
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:        []
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:      0
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:      0
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:             0
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##             - autoscaling        2
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##             - update             1
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 1648951 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##             - STARTED         1
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestTriggerIntegration -Dtests.method=testNodeMarkersRegistration 
-Dtests.seed=6D1EADCA5A32E1AF -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=mk -Dtests.timezone=Africa/Bamako -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.43s J2 | 
TestTriggerIntegration.testNodeMarkersRegistration <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Path 
/autoscaling/nodeAdded/127.0.0.1:10010_solr should have been deleted
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:75A425C654072C40]:0)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1649017 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 1649017 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 1649017 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10008_solr
   [junit4]   2> 1649017 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1649018 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1649021 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 6
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1649021 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649033 DEBUG (AutoscalingActionExecutor-4618-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"4111a8e880a95T38bxngjjsw1zhb3lymth2blvb",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144705664944789,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144705664944789],
   [junit4]   2>     "_enqueue_time_":1144707060507039,
   [junit4]   2>     "nodeNames":["127.0.0.1:10010_solr"]}}
   [junit4]   2> 1649033 WARN  (AutoscalingActionExecutor-4618-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_trigger has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1649033 DEBUG (AutoscalingActionExecutor-4618-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 22 ms for event 
id=4111a8e880a95T38bxngjjsw1zhb3lymth2blvb
   [junit4]   2> 1649042 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649068 DEBUG (ScheduledTrigger-4620-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649088 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649124 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649141 DEBUG (simCloudManagerPool-4619-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1649141 DEBUG (simCloudManagerPool-4619-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1649142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 1649142 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1649142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10011_solr at time 
1144721211178189
   [junit4]   2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649177 DEBUG (simCloudManagerPool-4619-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr, 127.0.0.1:10011_solr]
   [junit4]   2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10007_solr, 127.0.0.1:10011_solr]
   [junit4]   2> 1649178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 1649178 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1649178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing 
registered processor for nodes: [127.0.0.1:10011_solr] added at times 
[1144721211178189], now=1144722626030189
   [junit4]   2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":1144721211178189,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144721211178189],
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_restore_trigger]
   [junit4]   2> 1649197 DEBUG (AutoscalingActionExecutor-4621-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":1144721211178189,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144721211178189],
   [junit4]   2>     "_enqueue_time_":1144722645272739,
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:            3
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:   3
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:            0
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:   []
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:                0
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - autoscaling        
2
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - update             
1
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: 
node_added_restore_trigger
   [junit4]   2> 1649198 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - STARTED         1
   [junit4]   2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 
100ms
   [junit4]   2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 246 ms for event 
id=4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd
   [junit4]   2> 1649215 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventFromRestoredState
   [junit4]   2> 1649215 DEBUG 
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 1649215 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1649215 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1649216 DEBUG 
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 10
   [junit4]   2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10007_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
10
   [junit4]   2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1649227 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649247 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649303 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649327 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649336 DEBUG (simCloudManagerPool-4622-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1649336 DEBUG (simCloudManagerPool-4622-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 1649359 DEBUG (ScheduledTrigger-4623-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10007_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10007_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1649360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
11
   [junit4]   2> 1649360 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1649360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 
1144731577217889
   [junit4]   2> 1649379 DEBUG (ScheduledTrigger-4623-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649380 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649380 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649400 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649426 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649426 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649444 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649446 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649446 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649464 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649466 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649466 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649497 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649497 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649497 DEBUG (ScheduledTrigger-4623-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649517 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649517 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649517 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1649598 DEBUG (ScheduledTrigger-4623-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[1144731577217889], now=1144744501802389
   [junit4]   2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"411209705f361T38bxngjjsw1zhb3lymth2blvg",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144731577217889,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144731577217889],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger]
   [junit4]   2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"411209705f361T38bxngjjsw1zhb3lymth2blvg",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144731577217889,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144731577217889],
   [junit4]   2>     "_enqueue_time_":1144744527782839,
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 21 ms for event 
id=411209705f361T38bxngjjsw1zhb3lymth2blvg
   [junit4]   2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10013_solr at time 
1144749589232989
   [junit4]   2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649768 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649769 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649769 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649829 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1649829 DEBUG 
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1649830 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 11
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10013_solr, 
127.0.0.1:10012_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr]
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10013_solr, 
127.0.0.1:10012_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr]
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion -1
   [junit4]   2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
11
   [junit4]   2> 1649891 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1649891 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10013_solr
   [junit4]   2> 1649891 DEBUG (ScheduledTrigger-4626-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649906 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 1649906 DEBUG (ScheduledTrigger-4626-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649906 DEBUG (ScheduledTrigger-4626-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649931 DEBUG (ScheduledTrigger-4626-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10013_solr] added at times 
[1144749589232989], now=1144760139416589
   [junit4]   2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144749589232989,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144749589232989],
   [junit4]   2>     "nodeNames":["127.0.0.1:10013_solr"]}}
   [junit4]   2> 1649932 DEBUG (ScheduledTrigger-4626-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger]
   [junit4]   2> 1649932 DEBUG (AutoscalingActionExecutor-4627-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":1144749589232989,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144749589232989],
   [junit4]   2>     "_enqueue_time_":1144760156655139,
   [junit4]   2>     "nodeNames":["127.0.0.1:10013_solr"]}}
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testEventFromRestoredState
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:          4
   [junit4]   2> 1649932 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 4
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:          0
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: []
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:       0
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:       0
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:              0
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - autoscaling        1
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - update             3
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - STARTED         2
   [junit4]   2> 1649933 INFO  
(TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - SUCCEEDED       1
   [junit4]   2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 162 ms for event 
id=41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj
   [junit4]   2> 1649936 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 1649936 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 1649936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
12
   [junit4]   2> 1649936 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1649936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1649937 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1649937 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 13
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1
   [junit4]   2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
13
   [junit4]   2> 1649938 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1649938 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 1649938 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1649967 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650004 DEBUG (ScheduledTrigger-4629-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650032 DEBUG (ScheduledTrigger-4629-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650052 DEBUG (ScheduledTrigger-4629-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650077 DEBUG (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1650077 DEBUG (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
14
   [junit4]   2> 1650088 DEBUG (ScheduledTrigger-4629-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1650105 INFO  (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}],
 enabled=true}
   [junit4]   2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
14
   [junit4]   2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10014_solr at time 
1144768859523039
   [junit4]   2> 1650121 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10014_solr] added at times 
[1144768859523039], now=1144769875652839
   [junit4]   2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"]}}
   [junit4]   2> 1650127 DEBUG (ScheduledTrigger-4629-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger1]
   [junit4]   2> 1650141 DEBUG (AutoscalingActionExecutor-4630-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "_enqueue_time_":1144769902357589,
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"]}}
   [junit4]   2> 1650141 INFO  (AutoscalingActionExecutor-4630-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "_enqueue_time_":1144769902357589,
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"]}}
   [junit4]   2> 1650142 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1650142 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1650142 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1650143 WARN  (AutoscalingActionExecutor-4630-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_trigger1 has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1650144 DEBUG (AutoscalingActionExecutor-4630-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 115 ms for event 
id=411294538c7dfT38bxngjjsw1zhb3lymth2blvm
   [junit4]   2> 1650153 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 14
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10014_solr]
   [junit4]   2> 1650154 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10014_solr]
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}],
 enabled=true}
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
14
   [junit4]   2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10011_solr
   [junit4]   2> 1650157 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1650157 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 1650157 DEBUG (ScheduledTrigger-4632-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650157 DEBUG (ScheduledTrigger-4632-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr
   [junit4]   2> 1650157 DEBUG (ScheduledTrigger-4632-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "_dequeue_time_":1144771417388089,
   [junit4]   2>     "_enqueue_time_":1144769902357589,
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 1650157 DEBUG (ScheduledTrigger-4632-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger1]
   [junit4]   2> 1650166 DEBUG (AutoscalingActionExecutor-4633-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "_dequeue_time_":1144771417388089,
   [junit4]   2>     "_enqueue_time_":1144769902357589,
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 1650166 DEBUG (ScheduledTrigger-4632-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1650166 INFO  (AutoscalingActionExecutor-4633-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1144768859523039,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1144768859523039],
   [junit4]   2>     "_dequeue_time_":1144771417388089,
   [junit4]   2>     "_enqueue_time_":1144769902357589,
   [junit4]   2>     "nodeNames":["127.0.0.1:10014_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 1650166 DEBUG (ScheduledTrigger-4632-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 87 ms for event 
id=411294538c7dfT38bxngjjsw1zhb3lymth2blvm
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:              2
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:     2
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:              1
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          127.0.0.1:10011_solr
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:     []
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:   0
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:   0
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:          0
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - autoscaling        1
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - update             3
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - STARTED         2
   [junit4]   2> 1650259 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - SUCCEEDED       1
   [junit4]   2> 1650264 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 1650264 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
15
   [junit4]   2> 1650264 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10011_solr
   [junit4]   2> 1650265 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1650265 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1650265 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 16
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10014_solr]
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion -1
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
16
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 1650282 DEBUG (ScheduledTrigger-4635-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650315 DEBUG (ScheduledTrigger-4635-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650372 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650393 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650425 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650445 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650448 DEBUG (simCloudManagerPool-4634-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1650448 DEBUG (simCloudManagerPool-4634-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
17
   [junit4]   2> 1650452 DEBUG (simCloudManagerPool-4634-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10014_solr]
   [junit4]   2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
17
   [junit4]   2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17
   [junit4]   2> 1650465 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650485 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
18
   [junit4]   2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 
127.0.0.1:10014_solr]
   [junit4]   2> 1650538 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
18
   [junit4]   2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 1650558 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650647 DEBUG (ScheduledTrigger-4635-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650738 DEBUG (ScheduledTrigger-4635-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650773 DEBUG (ScheduledTrigger-4635-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1650775 DEBUG (ScheduledTrigger-4635-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"4112e547a9ac5T38bxngjjsw1zhb3lymth2blvq",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":1144790590331589,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10012_solr":250.0,
   [junit4]   2>       "127.0.0.1:10014_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10012_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10014_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1m

[...truncated too long message...]

lrJmxReporter@2ed6b9ac: rootName = solr_33999, domain = 
solr.core..system.shard1.replica_n9, service url = null, agent id = null] for 
registry solr.core..system.shard1.replica_n9 / 
com.codahale.metrics.MetricRegistry@116dc80a
   [junit4]   2> 3727314 INFO  (coreCloseExecutor-10184-thread-2) 
[n:127.0.0.1:42850_solr c:AutoscalingHistoryHandlerTest_collection s:shard1 
r:core_node8 x:AutoscalingHistoryHandlerTest_collection_shard1_replica_n7] 
o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.collection.AutoscalingHistoryHandlerTest_collection.shard1.leader,
 tag=376278702
   [junit4]   2> 3727314 INFO  (jetty-closer-4750-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@719dd81b{/solr,null,UNAVAILABLE}
   [junit4]   2> 3727315 INFO  (jetty-closer-4750-thread-2) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 3727319 INFO  (jetty-closer-4750-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@5c42805{/solr,null,UNAVAILABLE}
   [junit4]   2> 3727321 INFO  (jetty-closer-4750-thread-3) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 3727324 INFO  (coreCloseExecutor-10185-thread-2) 
[n:127.0.0.1:33999_solr c:.system s:shard1 r:core_node10 
x:.system_shard1_replica_n9] o.a.s.m.SolrMetricManager Closing metric reporters 
for registry=solr.collection..system.shard1.leader, tag=42380608
   [junit4]   2> 3727333 INFO  (jetty-closer-4750-thread-1) [    ] 
o.a.s.c.Overseer Overseer 
(id=99396511382437897-127.0.0.1:33999_solr-n_0000000000) closing
   [junit4]   2> 3727333 INFO  
(OverseerStateUpdate-99396511382437897-127.0.0.1:33999_solr-n_0000000000) 
[n:127.0.0.1:33999_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:33999_solr
   [junit4]   2> 3727334 WARN  
(OverseerAutoScalingTriggerThread-99396511382437897-127.0.0.1:33999_solr-n_0000000000)
 [n:127.0.0.1:33999_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3727335 DEBUG (jetty-closer-4750-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3727337 INFO  (jetty-closer-4750-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@5c28eab1{/solr,null,UNAVAILABLE}
   [junit4]   2> 3727337 INFO  (jetty-closer-4750-thread-1) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 3727342 ERROR 
(SUITE-AutoscalingHistoryHandlerTest-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3727343 INFO  
(SUITE-AutoscalingHistoryHandlerTest-seed#[6D1EADCA5A32E1AF]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:37461 37461
   [junit4]   2> 3727423 INFO  (Thread-5362) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:37461 37461
   [junit4]   2> 3727424 WARN  (Thread-5362) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        13      /solr/aliases.json
   [junit4]   2>        5       /solr/configs/.system
   [junit4]   2>        5       /solr/configs/.system/managed-schema
   [junit4]   2>        4       /solr/security.json
   [junit4]   2>        4       
/solr/configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/managed-schema
   [junit4]   2>        4       
/solr/configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        56      /solr/collections/.system/state.json
   [junit4]   2>        43      
/solr/collections/AutoscalingHistoryHandlerTest_collection/state.json
   [junit4]   2>        13      /solr/clusterprops.json
   [junit4]   2>        13      /solr/clusterstate.json
   [junit4]   2>        3       
/solr/overseer_elect/election/99396511382437897-127.0.0.1:33999_solr-n_0000000000
   [junit4]   2>        2       
/solr/collections/.system/leader_elect/shard1/election/99396511382437895-core_node6-n_0000000000
   [junit4]   2>        2       
/solr/collections/AutoscalingHistoryHandlerTest_collection/leader_elect/shard1/election/99396511382437897-core_node5-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        13      /solr/live_nodes
   [junit4]   2>        13      /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_6D1EADCA5A32E1AF-001
   [junit4]   2> Jan 23, 2018 1:20:39 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@649d4078),
 locale=es-US, timezone=UCT
   [junit4]   2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=131368280,total=519569408
   [junit4]   2> NOTE: All tests run in this JVM: 
[VMParamsZkACLAndCredentialsProvidersTest, HdfsCollectionsAPIDistributedZkTest, 
PropertiesRequestHandlerTest, TestCloudPseudoReturnFields, 
TestAnalyzeInfixSuggestions, TestQuerySenderListener, BadCopyFieldTest, 
SearchHandlerTest, TestLocalFSCloudBackupRestore, ClusterStateUpdateTest, 
TestReRankQParserPlugin, GraphQueryTest, TestFilteredDocIdSet, 
TestAddFieldRealTimeGet, ComputePlanActionTest, SolrShardReporterTest, 
TestSolrCloudWithDelegationTokens, TestPrepRecovery, TestSQLHandlerNonCloud, 
ResponseHeaderTest, TestSolrDeletionPolicy2, TestImplicitCoreProperties, 
SchemaApiFailureTest, TestSolrCoreProperties, NumericFieldsTest, 
TestCodecSupport, DistanceUnitsTest, TestRequestForwarding, ZkStateWriterTest, 
IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestReqParamsAPI, 
HdfsRestartWhileUpdatingTest, SpellCheckComponentTest, TestLegacyField, 
TermVectorComponentTest, SpatialHeatmapFacetsTest, ClusterStateTest, 
LeaderFailureAfterFreshStartTest, TestEmbeddedSolrServerConstructors, 
MergeStrategyTest, TestUpdate, TestCloudInspectUtil, NotRequiredUniqueKeyTest, 
PreAnalyzedFieldTest, ExplicitHLLTest, TestExactSharedStatsCache, 
PeerSyncReplicationTest, SpellPossibilityIteratorTest, 
TestSchemaVersionResource, TestFaceting, TestSearcherReuse, 
DistributedFacetPivotLargeTest, TestDocBasedVersionConstraints, JvmMetricsTest, 
TestCollationFieldDocValues, AddBlockUpdateTest, 
TestSolrCloudWithHadoopAuthPlugin, TestCoreDiscovery, TestBulkSchemaConcurrent, 
TestTolerantSearch, MultiTermTest, TestFieldResource, TestInitQParser, 
TestRealTimeGet, TestAtomicUpdateErrorCases, StatsReloadRaceTest, 
ConnectionReuseTest, TestMissingGroups, TestSystemCollAutoCreate, 
TestQueryTypes, SolrGangliaReporterTest, HLLUtilTest, TestNRTOpen, 
TestDistributedMissingSort, BasicDistributedZk2Test, TestXmlQParser, 
CircularListTest, TestStressCloudBlindAtomicUpdates, TestMacroExpander, 
LegacyCloudClusterPropTest, ShowFileRequestHandlerTest, AliasIntegrationTest, 
TestPivotHelperCode, TestComputePlanAction, BasicAuthIntegrationTest, 
ClassificationUpdateProcessorTest, CleanupOldIndexTest, LukeRequestHandlerTest, 
DocValuesMultiTest, DateFieldTest, TestFastWriter, CollectionsAPISolrJTest, 
CoreMergeIndexesAdminHandlerTest, TestFiltering, TestShortCircuitedRequests, 
TestValueSourceCache, CloudMLTQParserTest, LeaderElectionTest, 
TestFieldCacheReopen, TestUseDocValuesAsStored2, TestTlogReplica, 
CurrencyRangeFacetCloudTest, HdfsNNFailoverTest, DeleteStatusTest, 
TestSubQueryTransformerDistrib, MultiThreadedOCPTest, TestTriggerIntegration, 
TestOverriddenPrefixQueryForCustomFieldType, SolrCmdDistributorTest, 
CollectionReloadTest, TestSolr4Spatial2, AtomicUpdatesTest, 
CoreAdminHandlerTest, CopyFieldTest, TestCollectionAPI, SpatialFilterTest, 
ConvertedLegacyTest, TestQueryWrapperFilter, UnloadDistributedZkTest, 
DirectoryFactoryTest, UninvertDocValuesMergePolicyTest, 
DistribJoinFromCollectionTest, DocValuesTest, ChaosMonkeySafeLeaderTest, 
SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, 
ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, 
TestReplicationHandler, TestRandomDVFaceting, 
DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, 
TestStressReorder, TestStressVersions, DistributedTermsComponentTest, 
TestCoreContainer, TestSolr4Spatial, QueryElevationComponentTest, PeerSyncTest, 
BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, 
HighlighterTest, SoftAutoCommitTest, CSVRequestHandlerTest, TestComponentsName, 
SearchHandlerTest, UpdateParamsTest, TestQuerySenderNoQuery, 
ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, 
TestStressRecovery, CursorPagingTest, CloudExitableDirectoryReaderTest, 
CollectionStateFormat2Test, ConfigSetsAPITest, CreateCollectionCleanupTest, 
DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, 
HealthCheckHandlerTest, HttpPartitionTest, MoveReplicaTest, 
OutOfBoxZkACLAndCredentialsProvidersTest, 
OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, 
OverseerRolesTest, RecoveryAfterSoftCommitTest, TestCloudRecovery, 
TestLockTree, TestOnReconnectListenerSupport, TestRebalanceLeaders, 
ZkFailoverTest, TestCollectionsAPIViaSolrCloudCluster, 
AutoAddReplicasIntegrationTest, SearchRateTriggerTest, TestNodeAddedTrigger, 
TestNodeLostTrigger, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, 
CdcrVersionReplicationTest, HdfsSyncSliceTest, HdfsThreadLeakTest, 
HdfsTlogReplayBufferedWhileIndexingTest, HdfsUnloadDistributedZkTest, 
HdfsWriteToMultipleCollectionsTest, StressHdfsTest, 
ConfigureRecoveryStrategyTest, QueryResultKeyTest, TestConfigSetProperties, 
TestCorePropertiesReload, TestReloadAndDeleteDocs, TestSolrCoreSnapshots, 
TestConfigReload, AutoscalingHistoryHandlerTest]
   [junit4] Completed [654/768 (2!)] on J2 in 39.46s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 48487 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to