Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/373/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction.testIntegration

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
        at 
__randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:74EA751760421C57]:0)
        at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
        at java.util.ArrayList$Itr.next(ArrayList.java:859)
        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
        at 
__randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:4CB6F2447FBD5CDF]:0)
        at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
        at java.util.ArrayList$Itr.next(ArrayList.java:859)
        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
        at sun.reflect.GeneratedMethodAccessor33.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testGammaDistribution

Error Message:


Stack Trace:
java.lang.AssertionError
        at 
__randomizedtesting.SeedInfo.seed([1918B1C95D2400E1:24629A677E5CAAF6]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertTrue(Assert.java:54)
        at 
org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testGammaDistribution(StreamExpressionTest.java:7808)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11936 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestExecutePlanAction_C48B7B3B457DBD72-001/init-core-data-001
   [junit4]   2> 678206 INFO  
(SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 678206 INFO  
(SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 678206 INFO  
(SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 678235 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678235 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 678258 DEBUG (ScheduledTrigger-1711-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678260 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testExecute
   [junit4]   2> 678260 DEBUG 
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 678260 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 678264 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 678264 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 678269 DEBUG 
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 678269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 678270 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678290 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678314 DEBUG (ScheduledTrigger-1714-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678337 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678358 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678373 DEBUG (simCloudManagerPool-1713-thread-1) [    ] 
o.a.s.c.o.ClusterStateMutator building a new cName: testExecute
   [junit4]   2> 678378 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678386 DEBUG (simCloudManagerPool-1713-thread-1) [    ] 
o.a.s.c.Assign Identify nodes using default
   [junit4]   2> 678398 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678416 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction Collection ready after 1156ms
   [junit4]   2> 678418 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678420 DEBUG 
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"1506a604cef519c0T5y74lvmt9hwilvifly8sh3hko",
   [junit4]   2>   "source":"mock_trigger_name",
   [junit4]   2>   "eventTime":1515080864239000000,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1515080864239000000],
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}} with context 
properties: 
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@57fba7cb,
 org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction$1@74d02a4d]}
   [junit4]   2> 678420 DEBUG 
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: 
action=MOVEREPLICA&collection=testExecute&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node1
   [junit4]   2> 678438 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678464 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678484 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678504 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678521 DEBUG (simCloudManagerPool-1713-thread-4) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node4":{
   [junit4]   2>     "core":"testExecute_shard1_replica_n3",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testExecute",
   [junit4]   2>     "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 678524 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678550 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678574 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678596 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678616 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678636 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678639 DEBUG 
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: action=OVERSEERSTATUS
   [junit4]   2> 678643 DEBUG (simCloudManagerPool-1713-thread-5) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testExecute / 
shard1: {"core_node2":{
   [junit4]   2>     "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>     "core":"testExecute_shard1_replica_n2",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":123450000,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testExecute"}}
   [junit4]   2> 678644 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction Collection ready after 7ms
   [junit4]   2> 678644 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction -------------_ FINAL STATE --------------
   [junit4]   2> 678644 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Node values: {
   [junit4]   2>   "127.0.0.1:10001_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":2,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":980,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10001_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10001,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     
"metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
   [junit4]   2>   "127.0.0.1:10000_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":0,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":1000,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10000_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10000,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     
"metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
   [junit4]   2> 678644 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Live nodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Collection testExecute state: 
DocCollection(testExecute//clusterstate.json/3)={
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "autoCreated":"true",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "replicas":{
   [junit4]   2>         "core_node2":{
   [junit4]   2>           "core":"testExecute_shard1_replica_n2",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT"},
   [junit4]   2>         "core_node4":{
   [junit4]   2>           "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>           "core":"testExecute_shard1_replica_n3",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "type":"NRT"}},
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active"}}}
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testExecute
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 678645 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:           2
   [junit4]   2> 678646 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:  1
   [junit4]   2> 678646 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:           0
   [junit4]   2> 678647 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:  [testExecute]
   [junit4]   2> 678647 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:        2
   [junit4]   2> 678647 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:        0
   [junit4]   2> 678647 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:               2
   [junit4]   2> 678647 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * testExecute         2
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - active           2
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - CREATE             1
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - DELETESTATUS       2
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - MOVEREPLICA        1
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - OVERSEERSTATUS     1
   [junit4]   2> 678648 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##               - REQUESTSTATUS      2
   [junit4]   2> 678649 INFO  
(TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 678658 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678680 DEBUG (ScheduledTrigger-1714-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678688 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testIntegration
   [junit4]   2> 678688 DEBUG 
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 678688 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678693 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 678693 DEBUG 
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 4
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion -1
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 678694 DEBUG (ScheduledTrigger-1717-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678715 DEBUG (ScheduledTrigger-1717-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678735 DEBUG (ScheduledTrigger-1717-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678760 DEBUG (ScheduledTrigger-1717-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678780 DEBUG (ScheduledTrigger-1717-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678800 DEBUG (ScheduledTrigger-1717-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678813 DEBUG (simCloudManagerPool-1716-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 678813 DEBUG (simCloudManagerPool-1716-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
5
   [junit4]   2> 678816 DEBUG (simCloudManagerPool-1716-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 678816 DEBUG (simCloudManagerPool-1716-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 678816 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 678817 DEBUG (simCloudManagerPool-1716-thread-2) [    ] 
o.a.s.c.o.ClusterStateMutator building a new cName: testIntegration
   [junit4]   2> 678817 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678817 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 678817 DEBUG (ScheduledTrigger-1717-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 2
   [junit4]   2> 678820 DEBUG (ScheduledTrigger-1717-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678827 DEBUG (simCloudManagerPool-1716-thread-2) [    ] 
o.a.s.c.Assign Identify nodes using default
   [junit4]   2> 678841 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 678841 DEBUG (ScheduledTrigger-1717-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 2
   [junit4]   2> 678858 DEBUG (simCloudManagerPool-1716-thread-5) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testIntegration / 
shard1: {"core_node1":{
   [junit4]   2>     "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>     "core":"testIntegration_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":123450000,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testIntegration"}}
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10001_solr]
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}}
   [junit4]   2> 678885 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}}
   [junit4]   2> 678886 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}} with context 
properties: {BEFORE_ACTION=[compute_plan]}
   [junit4]   2> 678887 INFO  (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction Computed Plan: 
action=MOVEREPLICA&collection=testIntegration&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
   [junit4]   2> 678887 INFO  (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2824767046 
sessionWrapper.createTime 2824767035468342, this.sessionWrapper.createTime 
2824767035468342 
   [junit4]   2> 678887 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}} with context 
properties: 
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@760b1e58],
 AFTER_ACTION=[compute_plan], BEFORE_ACTION=[compute_plan, execute_plan]}
   [junit4]   2> 678887 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: 
action=MOVEREPLICA&collection=testIntegration&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
   [junit4]   2> 678994 DEBUG (simCloudManagerPool-1716-thread-7) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node4":{
   [junit4]   2>     "core":"testIntegration_shard1_replica_n3",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testIntegration",
   [junit4]   2>     "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 679111 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction -------------_ FINAL STATE --------------
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Node values: {
   [junit4]   2>   "127.0.0.1:10001_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":1,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":970,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10001_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10001,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     
"metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
   [junit4]   2>   "127.0.0.1:10000_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":2,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":980,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10000_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10000,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     
"metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Live nodes: [127.0.0.1:10000_solr]
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.TestExecutePlanAction * Collection testIntegration state: 
DocCollection(testIntegration//clusterstate.json/10)={
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "autoCreated":"true",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "replicas":{
   [junit4]   2>         "core_node1":{
   [junit4]   2>           "core":"testIntegration_shard1_replica_n1",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT"},
   [junit4]   2>         "core_node4":{
   [junit4]   2>           "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>           "core":"testIntegration_shard1_replica_n3",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "type":"NRT"}},
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active"}}}
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testIntegration
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:               1
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:      0
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:               1
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           127.0.0.1:10001_solr
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:      [testIntegration]
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:    2
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:    2
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:           2
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * testIntegration         2
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - active           2
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 679112 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - CREATE             1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - DELETESTATUS       1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - MOVEREPLICA        1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - REQUESTSTATUS      1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - autoscaling        1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - update             1
   [junit4]   2> 679113 INFO  
(TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 679113 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 11382 ms for event 
id=a091c092378ecT5y74lvmt9hwilvifly8sh3hkp
   [junit4]   2> 679114 DEBUG (ScheduledTrigger-1717-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 679116 DEBUG (ScheduledTrigger-1717-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestExecutePlanAction -Dtests.method=testIntegration 
-Dtests.seed=C48B7B3B457DBD72 -Dtests.slow=true -Dtests.locale=ar-QA 
-Dtests.timezone=Africa/Gaborone -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.47s J1 | TestExecutePlanAction.testIntegration <<<
   [junit4]    > Throwable #1: java.util.ConcurrentModificationException
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:74EA751760421C57]:0)
   [junit4]    >        at 
java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
   [junit4]    >        at java.util.ArrayList$Itr.next(ArrayList.java:859)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 679116 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 679120 DEBUG 
(SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestExecutePlanAction_C48B7B3B457DBD72-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1709, maxMBSortInHeap=5.080522274153202, 
sim=RandomSimilarity(queryNorm=true): {}, locale=ar-QA, timezone=Africa/Gaborone
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 
(64-bit)/cpus=3,threads=1,free=144327640,total=323878912
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUCache, 
AddSchemaFieldsUpdateProcessorFactoryTest, TestMacroExpander, 
CloneFieldUpdateProcessorFactoryTest, AnalyticsQueryTest, SolrCoreTest, 
BlockJoinFacetDistribTest, JavabinLoaderTest, JvmMetricsTest, TestCharFilters, 
TestSolrCoreParser, TriggerIntegrationTest, PropertiesRequestHandlerTest, 
SpellingQueryConverterTest, TestReload, ChaosMonkeySafeLeaderTest, 
LargeFieldTest, ReplaceNodeTest, DocumentAnalysisRequestHandlerTest, 
TestAnalyzeInfixSuggestions, AssignTest, TestCorePropertiesReload, 
DebugComponentTest, CdcrVersionReplicationTest, SolrSlf4jReporterTest, 
TestPerFieldSimilarityWithDefaultOverride, TestConfigSetsAPIExclusivity, 
TestFieldCacheWithThreads, SystemInfoHandlerTest, 
HdfsAutoAddReplicasIntegrationTest, TestSolrIndexConfig, TestIndexSearcher, 
QueryEqualityTest, TestLockTree, CollectionsAPIAsyncDistributedZkTest, 
DocValuesTest, TestFileDictionaryLookup, TestFastOutputStream, 
DistributedFacetPivotSmallAdvancedTest, HdfsBasicDistributedZkTest, 
AutoAddReplicasPlanActionTest, TestCryptoKeys, TestFieldCacheSortRandom, 
SpellPossibilityIteratorTest, TestExecutePlanAction]
   [junit4] Completed [99/765 (1!)] on J1 in 0.95s, 2 tests, 1 error <<< 
FAILURES!

[...truncated 735 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_C48B7B3B457DBD72-001/init-core-data-001
   [junit4]   2> 1839217 WARN  
(SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1839217 INFO  
(SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1839218 INFO  
(SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1839218 INFO  
(SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1839221 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 1839221 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 1
   [junit4]   2> 1839221 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1839221 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839221 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839221 DEBUG (ScheduledTrigger-5649-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839222 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1839223 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839243 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839266 DEBUG (ScheduledTrigger-5652-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839286 DEBUG (ScheduledTrigger-5652-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839306 DEBUG (ScheduledTrigger-5652-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1839326 DEBUG (simCloudManagerPool-5651-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839326 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1839326 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839326 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 
2871230138595399
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1839357 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10007_solr
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing 
registered processor for nodes: [127.0.0.1:10007_solr] added at times 
[2871231699896449], now=2871231712000599
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":2871231699896449,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871231699896449],
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 1839358 DEBUG (AutoscalingActionExecutor-5653-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":2871231699896449,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871231699896449],
   [junit4]   2>     "_enqueue_time_":2871231718427699,
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:            3
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:   3
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:            0
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:   []
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:                0
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - autoscaling        
2
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##                - update             
1
   [junit4]   2> 1839359 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1839359 DEBUG (AutoscalingActionExecutor-5653-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 43 ms for event 
id=a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestTriggerIntegration 
-Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=C48B7B3B457DBD72 
-Dtests.slow=true -Dtests.locale=id-ID -Dtests.timezone=America/Guayaquil 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.14s J0 | 
TestTriggerIntegration.testNodeAddedTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.util.ConcurrentModificationException
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:4CB6F2447FBD5CDF]:0)
   [junit4]    >        at 
java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
   [junit4]    >        at java.util.ArrayList$Itr.next(ArrayList.java:859)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839367 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 1839367 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 1839368 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1839369 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1839369 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 6
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1839370 DEBUG (ScheduledTrigger-5655-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839390 DEBUG (ScheduledTrigger-5655-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839411 DEBUG (ScheduledTrigger-5655-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839433 DEBUG (ScheduledTrigger-5655-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839453 DEBUG (ScheduledTrigger-5655-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839473 DEBUG (simCloudManagerPool-5654-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839473 DEBUG (simCloudManagerPool-5654-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 1839473 DEBUG (ScheduledTrigger-5655-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1839474 DEBUG (simCloudManagerPool-5654-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10008_solr]
   [junit4]   2> 1839474 DEBUG (simCloudManagerPool-5654-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10008_solr]
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 1839474 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 1839474 DEBUG (ScheduledTrigger-5655-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 2
   [junit4]   2> 1839474 DEBUG (ScheduledTrigger-5655-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr
   [junit4]   2> 1839493 DEBUG (ScheduledTrigger-5655-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839494 DEBUG (ScheduledTrigger-5655-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr
   [junit4]   2> 1839502 DEBUG (ScheduledTrigger-5655-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 2
   [junit4]   2> 1839504 DEBUG (simCloudManagerPool-5654-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 1839505 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger 
with currently live nodes: 2
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10008_solr]
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2871237544369149,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871237544369149],
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1839506 DEBUG (AutoscalingActionExecutor-5656-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2871237544369149,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871237544369149],
   [junit4]   2>     "_enqueue_time_":2871239094614549,
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1839506 DEBUG (AutoscalingActionExecutor-5656-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 22 ms for event 
id=a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:             2
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:    2
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:             1
   [junit4]   2> 1839506 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         127.0.0.1:10008_solr
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:    []
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:  0
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:  0
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:         0
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - autoscaling        2
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - update             2
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_restore_trigger
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - STARTED         1
   [junit4]   2> 1839507 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - SUCCEEDED       1
   [junit4]   2> 1839511 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 1839511 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 1839511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 1839511 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10008_solr
   [junit4]   2> 1839511 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1839512 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1839512 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839512 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839513 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1839513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 10
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10005_solr]
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
10
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1839514 DEBUG (ScheduledTrigger-5658-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839538 DEBUG (ScheduledTrigger-5658-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839559 DEBUG (ScheduledTrigger-5658-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839579 DEBUG (ScheduledTrigger-5658-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839599 DEBUG (ScheduledTrigger-5658-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839619 DEBUG (ScheduledTrigger-5658-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839620 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1
   [junit4]   2> 1839620 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1839620 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1839621 DEBUG 
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1839621 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 10
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
10
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10005_solr
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1839622 DEBUG (ScheduledTrigger-5661-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839622 DEBUG (ScheduledTrigger-5661-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
   [junit4]   2> 1839642 DEBUG (ScheduledTrigger-5661-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839662 DEBUG (ScheduledTrigger-5661-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839693 DEBUG (ScheduledTrigger-5661-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839722 DEBUG (ScheduledTrigger-5661-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1839722 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS
   [junit4]   2> 1839724 DEBUG (simCloudManagerPool-5660-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839724 DEBUG (simCloudManagerPool-5660-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 1839725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
11
   [junit4]   2> 1839725 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 1839725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 1839725 DEBUG (ScheduledTrigger-5661-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1839725 DEBUG (ScheduledTrigger-5661-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10009_solr]
   [junit4]   2> 1839726 DEBUG (simCloudManagerPool-5660-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}],
 enabled=true}
   [junit4]   2> 1839726 INFO  
(TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72])
 [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1
   [junit4]   2> 1839726 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
12
   [junit4]   2> 1839726 INFO  (Simu

[...truncated too long message...]

Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 143258 INFO  (jetty-closer-11-thread-4) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@757dfd3: rootName = 
solr_60604, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143260 INFO  
(zkCallback-42-thread-2-processing-n:127.0.0.1:60604_solr) 
[n:127.0.0.1:60604_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:60604_solr
   [junit4]   2> 143261 INFO  (jetty-closer-11-thread-4) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:60604_solr
   [junit4]   2> 143263 INFO  
(zkCallback-42-thread-2-processing-n:127.0.0.1:60604_solr) 
[n:127.0.0.1:60604_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 143263 INFO  (jetty-closer-11-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@7d7f82c{/solr,null,UNAVAILABLE}
   [junit4]   2> 143265 INFO  
(zkCallback-38-thread-1-processing-n:127.0.0.1:34995_solr) 
[n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 143267 INFO  (jetty-closer-11-thread-4) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 143270 INFO  
(zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) 
[n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 143319 INFO  (jetty-closer-11-thread-3) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 143319 INFO  (jetty-closer-11-thread-3) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@588bfbc3: rootName = 
solr_34995, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143320 INFO  
(zkCallback-38-thread-1-processing-n:127.0.0.1:34995_solr) 
[n:127.0.0.1:34995_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:34995_solr
   [junit4]   2> 143322 INFO  (jetty-closer-11-thread-3) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:34995_solr
   [junit4]   2> 143323 INFO  (jetty-closer-11-thread-3) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 143324 INFO  
(zkCallback-38-thread-2-processing-n:127.0.0.1:34995_solr) 
[n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 143325 INFO  
(zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) 
[n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) 
[n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 
x:collection1_shard2_replica_n3] o.a.s.c.SolrCore 
[collection1_shard2_replica_n3]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@1681d416
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) 
[n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 
x:collection1_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.core.collection1.shard2.replica_n3, tag=377607190
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) 
[n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 
x:collection1_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@3d610cca: rootName = 
solr_34995, domain = solr.core.collection1.shard2.replica_n3, service url = 
null, agent id = null] for registry solr.core.collection1.shard2.replica_n3 / 
com.codahale.metrics.MetricRegistry@4e9665be
   [junit4]   2> 143334 INFO  (coreCloseExecutor-274-thread-1) 
[n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 
x:collection1_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.collection1.shard2.leader, tag=377607190
   [junit4]   2> 143335 INFO  (jetty-closer-11-thread-4) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@410a5ffa{/solr,null,UNAVAILABLE}
   [junit4]   2> 143339 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, 
tag=null
   [junit4]   2> 143339 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@305ef: rootName = 
solr_50146, domain = solr.jvm, service url = null, agent id = null] for 
registry solr.jvm / com.codahale.metrics.MetricRegistry@4e4434e9
   [junit4]   2> 143343 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 143343 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@69f2d18b: rootName = 
solr_50146, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143345 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:50146_solr
   [junit4]   2> 143346 INFO  (jetty-closer-11-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 143346 INFO  
(zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) 
[n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 144860 WARN  
(zkCallback-38-thread-2-processing-n:127.0.0.1:34995_solr) 
[n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 144861 INFO  (jetty-closer-11-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@614b4ee6{/solr,null,UNAVAILABLE}
   [junit4]   2> 144862 INFO  (jetty-closer-11-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@e84c30{/solr,null,UNAVAILABLE}
   [junit4]   2> 144865 ERROR 
(SUITE-StreamExpressionTest-seed#[1918B1C95D2400E1]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 144867 INFO  
(SUITE-StreamExpressionTest-seed#[1918B1C95D2400E1]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:53399 53399
   [junit4]   2> 145021 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2>        at 
sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2>        at 
sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 145021 INFO  (Thread-52) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:53399 53399
   [junit4]   2> 145064 WARN  (Thread-52) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        99      /solr/aliases.json
   [junit4]   2>        17      /solr/configs/conf
   [junit4]   2>        8       /solr/configs/ml
   [junit4]   2>        4       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        99      /solr/clusterprops.json
   [junit4]   2>        99      /solr/clusterstate.json
   [junit4]   2>        38      
/solr/collections/destinationCollection/state.json
   [junit4]   2>        26      
/solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2>        20      
/solr/collections/parallelDestinationCollection/state.json
   [junit4]   2>        10      /solr/collections/mainCorpus1/state.json
   [junit4]   2>        10      /solr/collections/destination/state.json
   [junit4]   2>        10      /solr/collections/destination1/state.json
   [junit4]   2>        10      /solr/collections/collection1/state.json
   [junit4]   2>        10      /solr/collections/workQueue1/state.json
   [junit4]   2>        10      /solr/collections/workQueue/state.json
   [junit4]   2>        10      /solr/collections/uknownCollection/state.json
   [junit4]   2>        10      /solr/collections/modelCollection/state.json
   [junit4]   2>        10      
/solr/collections/checkpointCollection/state.json
   [junit4]   2>        10      /solr/collections/mainCorpus/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        99      /solr/live_nodes
   [junit4]   2>        99      /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_1918B1C95D2400E1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), 
sim=RandomSimilarity(queryNorm=false): {}, locale=fr-CH, timezone=Etc/GMT+12
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 
(64-bit)/cpus=3,threads=1,free=97033240,total=227909632
   [junit4]   2> NOTE: All tests run in this JVM: [FacetFieldTest, 
HttpSolrClientConPoolTest, CorrelationEvaluatorTest, CubedRootEvaluatorTest, 
ArcCosineEvaluatorTest, SolrSchemalessExampleTest, ModuloEvaluatorTest, 
StreamExpressionParserTest, TestPathTrie, AndEvaluatorTest, 
LargeVolumeJettyTest, StreamExpressionToExplanationTest, SolrParamTest, 
StreamExpressionTest]
   [junit4] Completed [70/146 (1!)] on J1 in 129.45s, 148 tests, 1 failure <<< 
FAILURES!

[...truncated 47457 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to