Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/440/

2 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration

Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10006_solr should have been deleted

Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10006_solr 
should have been deleted
        at 
__randomizedtesting.SeedInfo.seed([16D27FE08C18B5A:19D7AFF206F446B5]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertFalse(Assert.java:68)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:844)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory

Error Message:
expected:<5> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<5> but was:<0>
        at 
__randomizedtesting.SeedInfo.seed([16D27FE08C18B5A:6C918303B289745D]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:311)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12101 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 33107 INFO  
(SUITE-TestTriggerIntegration-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_16D27FE08C18B5A-001/init-core-data-001
   [junit4]   2> 33109 WARN  
(SUITE-TestTriggerIntegration-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 33109 INFO  
(SUITE-TestTriggerIntegration-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 33112 INFO  
(SUITE-TestTriggerIntegration-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 33147 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 33149 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 33149 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 33149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 33150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 1
   [junit4]   2> 33150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 33150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 33152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 33153 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33159 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 33159 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 33160 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33160 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 33160 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33160 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 33173 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33194 DEBUG (ScheduledTrigger-108-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33214 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33235 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33257 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33291 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33311 DEBUG (simCloudManagerPool-107-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 33311 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33314 DEBUG (simCloudManagerPool-107-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 33314 DEBUG (simCloudManagerPool-107-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 33315 DEBUG (simCloudManagerPool-107-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 33323 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 33323 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 33323 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33323 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 33323 DEBUG (ScheduledTrigger-108-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 3
   [junit4]   2> 33334 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33334 DEBUG (ScheduledTrigger-108-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 33358 DEBUG (ScheduledTrigger-108-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 2
   [junit4]   2> 33358 DEBUG (ScheduledTrigger-108-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 33358 DEBUG (ScheduledTrigger-108-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33372 DEBUG (simCloudManagerPool-107-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 33372 DEBUG (simCloudManagerPool-107-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 33373 DEBUG (simCloudManagerPool-107-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33373 DEBUG (simCloudManagerPool-107-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33373 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 33373 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 33373 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10002_solr
   [junit4]   2> 33373 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33373 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 33373 DEBUG (ScheduledTrigger-108-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 2
   [junit4]   2> 33373 DEBUG (ScheduledTrigger-108-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10002_solr]
   [junit4]   2> 33374 DEBUG (ScheduledTrigger-108-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b8807ab4571bTbkbktozd6313q2f8styh71o5v",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":19624635346999067,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624635346999067],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 33374 DEBUG (ScheduledTrigger-108-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_lost_restore_trigger]
   [junit4]   2> 33417 DEBUG (AutoscalingActionExecutor-109-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b8807ab4571bTbkbktozd6313q2f8styh71o5v",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":19624635346999067,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624635346999067],
   [junit4]   2>     "_enqueue_time_":19624635424156017,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:                2
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:       2
   [junit4]   2> 33419 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:                1
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##            127.0.0.1:10002_solr
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:       []
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:     0
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:     0
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:            0
   [junit4]   2> 33420 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 33421 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##            - autoscaling        2
   [junit4]   2> 33421 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##            - update             1
   [junit4]   2> 33421 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 33422 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_restore_trigger
   [junit4]   2> 33422 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##            - STARTED         1
   [junit4]   2> 33426 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 33426 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 33426 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 33427 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10002_solr
   [junit4]   2> 33429 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 33429 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 33429 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 33429 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33430 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:163)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 33471 WARN  (AutoscalingActionExecutor-109-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_lost_restore_trigger has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:523)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 33472 DEBUG (AutoscalingActionExecutor-109-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 2776 ms for event 
id=45b8807ab4571bTbkbktozd6313q2f8styh71o5v
   [junit4]   2> 33474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 33474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 6
   [junit4]   2> 33474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 33474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 33483 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33483 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 33484 DEBUG (ScheduledTrigger-111-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33504 DEBUG (ScheduledTrigger-111-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33558 DEBUG (ScheduledTrigger-111-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33578 DEBUG (ScheduledTrigger-111-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33584 DEBUG (simCloudManagerPool-110-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 33584 DEBUG (simCloudManagerPool-110-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 33597 DEBUG (simCloudManagerPool-110-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33597 DEBUG (simCloudManagerPool-110-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 33597 DEBUG (simCloudManagerPool-110-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33597 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 33597 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 33598 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 33598 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 33598 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 33599 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33619 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 33619 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 33619 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 
19624647651779567
   [junit4]   2> 33619 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33640 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 33640 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 33640 DEBUG (ScheduledTrigger-111-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33750 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 33751 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 33751 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing 
registered processor for nodes: [127.0.0.1:10003_solr] added at times 
[19624647651779567], now=19624654221327667
   [junit4]   2> 33751 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b883582063efTbkbktozd6313q2f8styh71o5x",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":19624647651779567,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624647651779567],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 33751 DEBUG (ScheduledTrigger-111-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_restore_trigger]
   [junit4]   2> 33753 DEBUG (simCloudManagerPool-110-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 33753 DEBUG (simCloudManagerPool-110-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 33753 DEBUG (simCloudManagerPool-110-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 33753 DEBUG (simCloudManagerPool-110-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 33753 DEBUG (simCloudManagerPool-110-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 33753 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 33772 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 33772 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 33777 DEBUG (AutoscalingActionExecutor-112-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b883582063efTbkbktozd6313q2f8styh71o5x",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":19624647651779567,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624647651779567],
   [junit4]   2>     "_enqueue_time_":19624654266748467,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:               3
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:      3
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:               0
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:      []
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:    0
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:    0
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:           0
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##           - autoscaling        2
   [junit4]   2> 33778 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##           - update             1
   [junit4]   2> 33779 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 33779 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: 
node_added_restore_trigger
   [junit4]   2> 33779 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##           - STARTED         1
   [junit4]   2> 33800 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 33800 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 33802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 33802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 33802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 33803 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 33803 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 33803 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 33804 WARN  (AutoscalingActionExecutor-112-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_restore_trigger has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:523)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 33804 DEBUG (AutoscalingActionExecutor-112-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 1383 ms for event 
id=45b883582063efTbkbktozd6313q2f8styh71o5x
   [junit4]   2> 33815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 33815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 10
   [junit4]   2> 33815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33816 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 33816 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 33829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 33829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 33832 DEBUG (ScheduledTrigger-114-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33852 DEBUG (ScheduledTrigger-114-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33873 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33893 DEBUG (ScheduledTrigger-114-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33913 DEBUG (ScheduledTrigger-114-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 33925 DEBUG (simCloudManagerPool-113-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 33926 DEBUG (simCloudManagerPool-113-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 33926 DEBUG (simCloudManagerPool-113-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33926 DEBUG (simCloudManagerPool-113-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 33926 DEBUG (simCloudManagerPool-113-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 33926 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 33927 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 33927 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 33927 DEBUG (ScheduledTrigger-114-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 33927 DEBUG (ScheduledTrigger-114-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 33935 DEBUG (ScheduledTrigger-114-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33947 DEBUG (ScheduledTrigger-114-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 33947 DEBUG (ScheduledTrigger-114-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 33947 DEBUG (ScheduledTrigger-114-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
19624664053516417
   [junit4]   2> 33955 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 33968 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 33968 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 33968 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[19624664053516417], now=19624665072050967
   [junit4]   2> 33968 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b88729bf0881Tbkbktozd6313q2f8styh71o5z",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":19624664053516417,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624664053516417],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 33971 DEBUG (ScheduledTrigger-114-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger]
   [junit4]   2> 34017 DEBUG (AutoscalingActionExecutor-115-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b88729bf0881Tbkbktozd6313q2f8styh71o5z",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":19624664053516417,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624664053516417],
   [junit4]   2>     "_enqueue_time_":19624665249936917,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 34051 DEBUG (AutoscalingActionExecutor-115-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 34051 DEBUG (AutoscalingActionExecutor-115-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 34051 DEBUG (AutoscalingActionExecutor-115-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 1696 ms for event 
id=45b88729bf0881Tbkbktozd6313q2f8styh71o5z
   [junit4]   2> 34055 DEBUG (simCloudManagerPool-113-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34056 DEBUG (simCloudManagerPool-113-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 34056 DEBUG (simCloudManagerPool-113-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 34056 DEBUG (simCloudManagerPool-113-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 34056 DEBUG (simCloudManagerPool-113-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 34056 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 34057 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:           3
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:  3
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:           0
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:  []
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:        0
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:        0
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:               0
   [junit4]   2> 34097 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - autoscaling        2
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - update             2
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - STARTED         1
   [junit4]   2> 34098 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - SUCCEEDED       1
   [junit4]   2> 34124 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 34124 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 13
   [junit4]   2> 34125 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 34125 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 34125 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 34132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 34132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 14
   [junit4]   2> 34132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10004_solr]
   [junit4]   2> 34132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
   [junit4]   2> 34132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 34139 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 34139 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 34139 DEBUG (ScheduledTrigger-117-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34159 DEBUG (ScheduledTrigger-117-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34184 DEBUG (ScheduledTrigger-117-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34207 DEBUG (ScheduledTrigger-117-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34227 DEBUG (ScheduledTrigger-117-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34233 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1
   [junit4]   2> 34233 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 34233 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 34236 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 34239 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 14
   [junit4]   2> 34240 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34240 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
   [junit4]   2> 34241 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 34242 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10004_solr
   [junit4]   2> 34243 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 34243 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 34243 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34244 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 34264 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34285 DEBUG (ScheduledTrigger-120-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34305 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34325 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34337 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS
   [junit4]   2> 34344 DEBUG (simCloudManagerPool-119-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34347 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34348 DEBUG (simCloudManagerPool-119-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
15
   [junit4]   2> 34354 DEBUG (simCloudManagerPool-119-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34354 DEBUG (simCloudManagerPool-119-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34354 DEBUG (simCloudManagerPool-119-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 34354 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 34355 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 34355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 34355 DEBUG (ScheduledTrigger-120-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 34355 DEBUG (ScheduledTrigger-120-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
16
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34356 DEBUG (simCloudManagerPool-119-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 34356 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 34357 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 34357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 34357 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1
   [junit4]   2> 34363 DEBUG (ScheduledTrigger-120-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 3
   [junit4]   2> 34367 DEBUG (ScheduledTrigger-120-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 34376 DEBUG (ScheduledTrigger-120-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 34376 DEBUG (ScheduledTrigger-120-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 34376 DEBUG (ScheduledTrigger-120-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10006_solr at time 
19624685471740917
   [junit4]   2> 34384 DEBUG (ScheduledTrigger-120-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 3
   [junit4]   2> 34387 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 34459 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 34459 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 34459 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10006_solr] added at times 
[19624685471740917], now=19624689640036217
   [junit4]   2> 34459 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b88c265f33f5Tbkbktozd6313q2f8styh71o62",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":19624685471740917,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624685471740917],
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:            3
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:   3
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:            1
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                127.0.0.1:10004_solr
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:   []
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:                0
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - autoscaling        
2
   [junit4]   2> 34460 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[16D27FE08C18B5A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestTriggerIntegration -Dtests.method=testNodeMarkersRegistration 
-Dtests.seed=16D27FE08C18B5A -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=sv-SE -Dtests.timezone=Europe/Nicosia -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4]   2> 34463 DEBUG (ScheduledTrigger-120-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_lost_trigger, node_added_trigger]
   [junit4] FAILURE 0.37s J1 | 
TestTriggerIntegration.testNodeMarkersRegistration <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Path 
/autoscaling/nodeAdded/127.0.0.1:10006_solr should have been deleted
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([16D27FE08C18B5A:19D7AFF206F446B5]:0)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:844)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 34485 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[16D27FE08C18B5A]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 34486 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 17
   [junit4]   2> 34486 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10004_solr
   [junit4]   2> 34486 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 34486 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[16D27FE08C18B5A]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 34486 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 34487 DEBUG (AutoscalingActionExecutor-121-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b88c265f33f5Tbkbktozd6313q2f8styh71o62",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":19624685471740917,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624685471740917],
   [junit4]   2>     "_enqueue_time_":19624689866932717,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 34487 WARN  (AutoscalingActionExecutor-121-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_trigger has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:523)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 34487 DEBUG (AutoscalingActionExecutor-121-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 127 ms for event 
id=45b88c265f33f5Tbkbktozd6313q2f8styh71o62
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 18
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion -1
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18
   [junit4]   2> 34488 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 34489 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 34489 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34509 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34530 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34550 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34570 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34651 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34666 DEBUG (simCloudManagerPool-122-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34666 DEBUG (simCloudManagerPool-122-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
19
   [junit4]   2> 34666 DEBUG (simCloudManagerPool-122-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34666 DEBUG (simCloudManagerPool-122-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34667 DEBUG (simCloudManagerPool-122-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19
   [junit4]   2> 34672 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 34672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19
   [junit4]   2> 34675 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34675 DEBUG (ScheduledTrigger-123-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 34675 DEBUG (ScheduledTrigger-123-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 34681 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34681 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
20
   [junit4]   2> 34681 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34681 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34681 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34682 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 34682 DEBUG (simCloudManagerPool-122-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20
   [junit4]   2> 34682 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 34682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 34682 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 34682 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 34682 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 
19624700811712367
   [junit4]   2> 34682 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing 
registered processor for nodes: [127.0.0.1:10007_solr] added at times 
[19624700811712367], now=19624700813530517
   [junit4]   2> 34683 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b88fb8b4976fTbkbktozd6313q2f8styh71o64",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":19624700811712367,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624700811712367],
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34691 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger2, node_added_trigger1]
   [junit4]   2> 34692 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b88fb8b4976fTbkbktozd6313q2f8styh71o64",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":19624700811712367,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624700811712367],
   [junit4]   2>     "_enqueue_time_":19624701234765617,
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34692 INFO  (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 34716 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 34716 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 34716 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 34717 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 1248 ms for event 
id=45b88fb8b4976fTbkbktozd6313q2f8styh71o64
   [junit4]   2> 34816 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 
19624707535326967
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10007_solr] added at times 
[19624707535326967], now=19624707536920017
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b8914976cef7Tbkbktozd6313q2f8styh71o67",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":19624707535326967,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624707535326967],
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34817 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger2, node_added_trigger1]
   [junit4]   2> 34818 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b8914976cef7Tbkbktozd6313q2f8styh71o67",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":19624707535326967,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624707535326967],
   [junit4]   2>     "_enqueue_time_":19624707548402767,
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34818 INFO  (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration last action at 19624701287950667 time = 
19624707588424367
   [junit4]   2> 34818 INFO  (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 34819 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 34819 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 34819 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 34819 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 41 ms for event 
id=45b8914976cef7Tbkbktozd6313q2f8styh71o67
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
21
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34830 DEBUG (simCloudManagerPool-122-thread-7) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34831 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 21
   [junit4]   2> 34831 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 34831 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 21, lastZnodeVersion 21
   [junit4]   2> 34831 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 3
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
22
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34836 DEBUG (simCloudManagerPool-122-thread-8) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 34836 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 22
   [junit4]   2> 34837 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 34837 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 22, lastZnodeVersion 22
   [junit4]   2> 34837 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 3
   [junit4]   2> 34851 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 2
   [junit4]   2> 34851 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10007_solr
   [junit4]   2> 34851 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10007_solr]
   [junit4]   2> 34851 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"45b891b00fa341Tbkbktozd6313q2f8styh71o6a",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":19624709256618817,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624709256618817],
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34852 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, 
.auto_add_replicas, node_added_trigger2, node_lost_trigger1, 
node_added_trigger1]
   [junit4]   2> 34856 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"45b891b00fa341Tbkbktozd6313q2f8styh71o6a",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":19624709256618817,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[19624709256618817],
   [junit4]   2>     "_enqueue_time_":19624709269112017,
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 34856 INFO  (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger1
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 34857 DEBUG (AutoscalingActionExecutor-124-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 53 ms for event 
id=45b891b00fa341Tbkbktozd6313q2f8styh71o6a
   [junit4]   2> 34957 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 34957 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10007_solr
   [junit4]   2> 34957 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 34957 DEBUG (ScheduledTrigger-123-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 34957 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 34958 DEBUG (ScheduledTrigger-123-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 34958 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 2
   [junit4]   2> 34958 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10007_solr
   [junit4]   2> 34958 DEBUG (ScheduledTrigger-123-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostT

[...truncated too long message...]

junit4]   2> 2329774 INFO  (coreCloseExecutor-10227-thread-3) 
[n:127.0.0.1:42126_solr c:AutoscalingHistoryHandlerTest_collection s:shard1 
r:core_node5 x:AutoscalingHistoryHandlerTest_collection_shard1_replica_n2] 
o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.collection.AutoscalingHistoryHandlerTest_collection.shard1.leader,
 tag=1536081803
   [junit4]   2> 2329775 INFO  (jetty-closer-6514-thread-2) [    ] 
o.a.s.c.Overseer Overseer 
(id=73343862572515333-127.0.0.1:42126_solr-n_0000000000) closing
   [junit4]   2> 2329776 INFO  
(OverseerStateUpdate-73343862572515333-127.0.0.1:42126_solr-n_0000000000) 
[n:127.0.0.1:42126_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:42126_solr
   [junit4]   2> 2329776 WARN  
(OverseerAutoScalingTriggerThread-73343862572515333-127.0.0.1:42126_solr-n_0000000000)
 [n:127.0.0.1:42126_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2329776 DEBUG (jetty-closer-6514-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 2329777 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x10491dabb170005, likely client has closed socket
   [junit4]   2> 2329781 INFO  (jetty-closer-6514-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1b706e9a{/solr,null,UNAVAILABLE}
   [junit4]   2> 2329781 INFO  (jetty-closer-6514-thread-2) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 2329781 ERROR 
(SUITE-AutoscalingHistoryHandlerTest-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2329781 INFO  
(SUITE-AutoscalingHistoryHandlerTest-seed#[16D27FE08C18B5A]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:60101 60101
   [junit4]   2> 2329953 INFO  (Thread-10982) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:60101 60101
   [junit4]   2> 2329953 WARN  (Thread-10982) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/aliases.json
   [junit4]   2>        4       
/solr/collections/AutoscalingHistoryHandlerTest_collection/terms/shard1
   [junit4]   2>        3       /solr/security.json
   [junit4]   2>        3       
/solr/configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/managed-schema
   [junit4]   2>        2       /solr/collections/.system/terms/shard1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/clusterstate.json
   [junit4]   2>        11      /solr/clusterprops.json
   [junit4]   2>        7       
/solr/collections/AutoscalingHistoryHandlerTest_collection/state.json
   [junit4]   2>        7       /solr/collections/.system/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/live_nodes
   [junit4]   2>        11      /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_16D27FE08C18B5A-001
   [junit4]   2> Feb 23, 2018 8:36:51 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{action_s=FSTOrd50, 
event.source_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), type=FSTOrd50, 
before.actions_ss=PostingsFormat(name=LuceneVarGapDocFreqInterval), 
event.property._enqueue_time__ss=FSTOrd50, 
after.actions_ss=PostingsFormat(name=MockRandom), stage_s=FSTOrd50, 
event.id_s=PostingsFormat(name=LuceneFixedGap), 
event.property.eventTimes_ss=FSTOrd50, 
source_s=PostingsFormat(name=LuceneFixedGap), event.type_s=FSTOrd50, 
id=PostingsFormat(name=LuceneFixedGap), event.property.nodeNames_ss=FSTOrd50}, 
docValues:{_version_=DocValuesFormat(name=Asserting), 
event.time_l=DocValuesFormat(name=Asserting), 
timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=457, 
maxMBSortInHeap=7.573683662405026, sim=RandomSimilarity(queryNorm=true): {}, 
locale=is-IS, timezone=Pacific/Majuro
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=254164976,total=523763712
   [junit4]   2> NOTE: All tests run in this JVM: [TestCryptoKeys, 
ClassificationUpdateProcessorFactoryTest, TestSchemaSimilarityResource, 
TestLuceneIndexBackCompat, CollectionsAPIAsyncDistributedZkTest, 
DirectUpdateHandlerTest, DebugComponentTest, TestHttpShardHandlerFactory, 
SuggesterWFSTTest, FieldAnalysisRequestHandlerTest, 
ExitableDirectoryReaderTest, OpenCloseCoreStressTest, TestSolrCoreParser, 
JSONWriterTest, ResponseBuilderTest, TestSolrCloudWithDelegationTokens, 
TestSortableTextField, UtilsToolTest, TolerantUpdateProcessorTest, 
TestFieldCollectionResource, SystemLogListenerTest, 
TermVectorComponentDistributedTest, TestCollapseQParserPlugin, 
TestLeaderElectionWithEmptyReplica, TestDocumentBuilder, OutputWriterTest, 
ImplicitSnitchTest, TestSimDistributedQueue, HdfsSyncSliceTest, 
BadIndexSchemaTest, TestMultiWordSynonyms, ShardRoutingTest, 
TestUniqueKeyFieldResource, TestSortByMinMaxFunction, 
TestShortCircuitedRequests, SolrIndexSplitterTest, SolrRequestParserTest, 
TestOmitPositions, AnalysisErrorHandlingTest, DistributedTermsComponentTest, 
ChaosMonkeySafeLeaderWithPullReplicasTest, TestCustomDocTransformer, 
DistribDocExpirationUpdateProcessorTest, TestHighFrequencyDictionaryFactory, 
TestSlowCompositeReaderWrapper, CheckHdfsIndexTest, TestCSVResponseWriter, 
V2StandaloneTest, FacetPivotSmallTest, TestConfigSetProperties, 
SolrIndexConfigTest, SolrCloudReportersTest, TestCloudSchemaless, 
TestNodeAddedTrigger, ZkNodePropsTest, JavabinLoaderTest, TestTlogReplica, 
TestComponentsName, TestFieldCacheWithThreads, SolrMetricsIntegrationTest, 
TestRTimerTree, AutoScalingHandlerTest, SolrCoreCheckLockOnStartupTest, 
FullHLLTest, TestRecoveryHdfs, SpellingQueryConverterTest, 
CloudExitableDirectoryReaderTest, BinaryUpdateRequestHandlerTest, 
DefaultValueUpdateProcessorTest, TestWordDelimiterFilterFactory, 
TestSolrCloudSnapshots, TestFieldTypeResource, TestFieldCacheSort, 
TestManagedSchema, TestStressUserVersions, ShufflingReplicaListTransformerTest, 
IndexBasedSpellCheckerTest, BigEndianAscendingWordSerializerTest, 
CdcrReplicationHandlerTest, TestSQLHandler, TestAuthenticationFramework, 
VMParamsZkACLAndCredentialsProvidersTest, BlockJoinFacetRandomTest, 
HdfsUnloadDistributedZkTest, CursorMarkTest, UUIDFieldTest, NumberUtilsTest, 
SimpleCollectionCreateDeleteTest, TestClusterStateProvider, 
OpenExchangeRatesOrgProviderTest, CursorPagingTest, 
PreAnalyzedFieldManagedSchemaCloudTest, AnalyticsMergeStrategyTest, 
FullSolrCloudDistribCmdsTest, TestSimpleQParserPlugin, TestCollectionAPIs, 
TestSort, MoveReplicaTest, TestPHPSerializedResponseWriter, 
HdfsChaosMonkeyNothingIsSafeTest, TestCSVLoader, 
PeerSyncWithIndexFingerprintCachingTest, TestCustomSort, 
TestInPlaceUpdatesStandalone, SubstringBytesRefFilterTest, TestSimpleTextCodec, 
SpellCheckCollatorTest, TestGraphTermsQParserPlugin, TestZkAclsWithHadoopAuth, 
RAMDirectoryFactoryTest, TestIBSimilarityFactory, 
CloneFieldUpdateProcessorFactoryTest, HttpPartitionOnCommitTest, 
AddSchemaFieldsUpdateProcessorFactoryTest, DistributedFacetExistsSmallTest, 
TestExecutePlanAction, LeaderInitiatedRecoveryOnShardRestartTest, 
TestUtilizeNode, TestNonDefinedSimilarityFactory, 
SegmentsInfoRequestHandlerTest, TestRandomFlRTGCloud, 
ConfigureRecoveryStrategyTest, TestGroupingSearch, 
TestSubQueryTransformerCrossCore, DistributedQueryElevationComponentTest, 
CdcrBidirectionalTest, SparseHLLTest, BasicDistributedZkTest, 
ChaosMonkeySafeLeaderTest, LeaderElectionIntegrationTest, BasicZkTest, 
RecoveryZkTest, TestReplicationHandler, TestRandomFaceting, LeaderElectionTest, 
ZkSolrClientTest, ZkCLITest, TestDistributedSearch, ZkControllerTest, 
TestRealTimeGet, TestReload, TestStressVersions, TestCoreContainer, 
SimpleFacetsTest, TestSolr4Spatial, StatsComponentTest, 
SpellCheckComponentTest, PeerSyncTest, ConvertedLegacyTest, TestFiltering, 
TestFunctionQuery, TestIndexSearcher, HighlighterTest, SimplePostToolTest, 
TestCoreDiscovery, SuggesterFSTTest, TestFoldingMultitermQuery, 
TestStressLucene, PolyFieldTest, TestPseudoReturnFields, 
FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, 
SortByFunctionTest, TestRemoteStreaming, TestSolrDeletionPolicy1, 
TestWriterPerf, DirectSolrSpellCheckerTest, TestQueryTypes, 
RequestHandlersTest, PathHierarchyTokenizerFactoryTest, 
TestIndexingPerformance, MoreLikeThisHandlerTest, TestJmxIntegration, 
ReturnFieldsTest, MBeansHandlerTest, PreAnalyzedFieldTest, DateFieldTest, 
DOMUtilTest, ClusterStateTest, TestSolrJ, TestUtils, SliceStateTest, 
SystemInfoHandlerTest, DistributedMLTComponentTest, TestRTGBase, 
DistributedIntervalFacetingTest, TestCrossCoreJoin, 
TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, 
TestSimpleTrackingShardHandler, TestEmbeddedSolrServerAdminHandler, 
TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, 
ActionThrottleTest, AddReplicaTest, 
ChaosMonkeyNothingIsSafeWithPullReplicasTest, DistributedVersionInfoTest, 
MissingSegmentRecoveryTest, MoveReplicaHDFSTest, RemoteQueryErrorTest, 
ReplaceNodeNoTargetTest, ReplicationFactorTest, RestartWhileUpdatingTest, 
RollingRestartTest, TestCloudPseudoReturnFields, TestCloudRecovery, 
TestPrepRecovery, TestStressInPlaceUpdates, 
HdfsCollectionsAPIDistributedZkTest, ShardSplitTest, TestShardHandlerFactory, 
TestSolrConfigHandler, TestConfigReload, V2ApiIntegrationTest, 
AutoscalingHistoryHandlerTest]
   [junit4] Completed [643/777 (2!)] on J0 in 18.86s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 50606 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to