Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.3/48/

3 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at 
__randomizedtesting.SeedInfo.seed([CA6E4E49ADA1E82A:4253C73697610987]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:368)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.TestLBHttpSolrClient.testReliability

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available 
to handle this request
        at 
__randomizedtesting.SeedInfo.seed([D9AF7EB640850D97:1867A3F0E1E3DC3E]:0)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:726)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:638)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at 
org.apache.solr.client.solrj.TestLBHttpSolrClient.testReliability(TestLBHttpSolrClient.java:219)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured 
while waiting response from server at: https://127.0.0.1:35011/solr/collection1
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:663)
        ... 44 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
        at sun.security.ssl.InputRecord.read(InputRecord.java:503)
        at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
        at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
        at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 47 more


FAILED:  
junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.JDBCStreamTest

Error Message:
Error from server at http://127.0.0.1:46079/solr: KeeperErrorCode = NoNode for 
/overseer/collection-queue-work/qnr-0000000000

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:46079/solr: KeeperErrorCode = NoNode for 
/overseer/collection-queue-work/qnr-0000000000
        at __randomizedtesting.SeedInfo.seed([D9AF7EB640850D97]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at 
org.apache.solr.client.solrj.io.stream.JDBCStreamTest.setupCluster(JDBCStreamTest.java:76)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12131 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 0    INFO  
(SUITE-TestTriggerIntegration-seed#[CA6E4E49ADA1E82A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.3/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_CA6E4E49ADA1E82A-001/init-core-data-001
   [junit4]   2> 726  INFO  
(SUITE-TestTriggerIntegration-seed#[CA6E4E49ADA1E82A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 980  INFO  
(SUITE-TestTriggerIntegration-seed#[CA6E4E49ADA1E82A]-worker) [    ] 
o.e.j.u.log Logging initialized @13055ms to org.eclipse.jetty.util.log.Slf4jLog
   [junit4]   2> 998  INFO  
(SUITE-TestTriggerIntegration-seed#[CA6E4E49ADA1E82A]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1309 INFO  
(SUITE-TestTriggerIntegration-seed#[CA6E4E49ADA1E82A]-worker) [    ] 
o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find 
system property or JNDI)
   [junit4] IGNOR/A 0.00s J0 | 
TestTriggerIntegration.testNodeLostTriggerRestoreState
   [junit4]    > Assumption #1: 'badapple' test group is disabled 
(@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
   [junit4]   2> 1996 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 2086 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 2087 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 2146 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 2163 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 2182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 2775 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 2775 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 2776 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 2784 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
2
   [junit4]   2> 2801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 2882 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 2882 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
   [junit4]   2> 2883 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 2883 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 2883 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 2901 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 2922 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 2975 DEBUG (ScheduledTrigger-6-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3000 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3036 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3073 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3101 DEBUG (simCloudManagerPool-5-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3109 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3137 DEBUG (simCloudManagerPool-5-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 3140 DEBUG (simCloudManagerPool-5-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 3140 DEBUG (simCloudManagerPool-5-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 3140 DEBUG (simCloudManagerPool-5-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 3140 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 3140 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3141 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 3146 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3219 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 3146 DEBUG (ScheduledTrigger-6-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 3220 DEBUG (ScheduledTrigger-6-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 3220 DEBUG (ScheduledTrigger-6-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 
5140888560842485
   [junit4]   2> 3255 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 3272 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 3304 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 3292 DEBUG (ScheduledTrigger-6-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 3309 DEBUG (simCloudManagerPool-5-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3330 DEBUG (simCloudManagerPool-5-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 3330 DEBUG (simCloudManagerPool-5-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 3330 DEBUG (simCloudManagerPool-5-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger 
instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 3341 DEBUG (simCloudManagerPool-5-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 3341 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 3365 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 3365 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing 
registered processor for nodes: [127.0.0.1:10002_solr] added at times 
[5140888560842485], now=5140895811079985
   [junit4]   2> 3366 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"12439c647fb6f5T15c3qus8f0ezck17grlu6ga27",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":5140888560842485,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140888560842485],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 3365 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 3367 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3367 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10002_solr
   [junit4]   2> 3367 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 3369 DEBUG (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_restore_trigger]
   [junit4]   2> 3382 ERROR (ScheduledTrigger-6-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Unexpected exception in NodeAddedTrigger
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_restore_trigger has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.enqueue(ScheduledTriggers.java:522)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$4(ScheduledTriggers.java:276)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.NodeAddedTrigger.run(NodeAddedTrigger.java:164)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.run(ScheduledTriggers.java:572)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
   [junit4]   2>        at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3471 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 3562 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 3562 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 3562 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 3562 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 3562 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:               3
   [junit4]   2> 3564 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:      3
   [junit4]   2> 3564 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:               0
   [junit4]   2> 3566 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:      []
   [junit4]   2> 3566 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:    0
   [junit4]   2> 3566 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:    0
   [junit4]   2> 3566 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:           0
   [junit4]   2> 3566 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 3589 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##           - autoscaling        2
   [junit4]   2> 3589 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA6E4E49ADA1E82A])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestTriggerIntegration 
-Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=CA6E4E49ADA1E82A 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr 
-Dtests.timezone=Australia/Tasmania -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.95s J0 | 
TestTriggerIntegration.testNodeAddedTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The trigger did not 
fire at all
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([CA6E4E49ADA1E82A:4253C73697610987]:0)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:368)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3642 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger
   [junit4]   2> 3642 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 3661 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 3662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 3678 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 3679 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:175)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3680 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 3727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 3727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
6
   [junit4]   2> 3727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 3727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 3728 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 3736 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3736 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 3736 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3771 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3796 DEBUG (ScheduledTrigger-9-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3824 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3825 DEBUG (simCloudManagerPool-8-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3826 DEBUG (simCloudManagerPool-8-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 3844 DEBUG (simCloudManagerPool-8-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 3844 DEBUG (ScheduledTrigger-9-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3844 DEBUG (simCloudManagerPool-8-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 3845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 3845 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 3845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 3846 DEBUG (ScheduledTrigger-9-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 2
   [junit4]   2> 3865 DEBUG (ScheduledTrigger-9-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 3865 DEBUG (ScheduledTrigger-9-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 3873 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 3873 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr
   [junit4]   2> 3885 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 3893 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 3894 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10002_solr]
   [junit4]   2> 3894 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243a3fe04f3abT15c3qus8f0ezck17grlu6ga28",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":5140921201259435,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140921201259435],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 3894 DEBUG (ScheduledTrigger-9-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_lost_trigger]
   [junit4]   2> 3918 DEBUG (AutoscalingActionExecutor-10-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243a3fe04f3abT15c3qus8f0ezck17grlu6ga28",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":5140921201259435,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140921201259435],
   [junit4]   2>     "_enqueue_time_":5140922252579535,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 3924 DEBUG (simCloudManagerPool-8-thread-3) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 3924 DEBUG (simCloudManagerPool-8-thread-3) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 3925 DEBUG (simCloudManagerPool-8-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
   [junit4]   2> 3925 DEBUG (simCloudManagerPool-8-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
   [junit4]   2> 3929 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 3939 DEBUG (AutoscalingActionExecutor-10-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 3939 DEBUG (AutoscalingActionExecutor-10-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
   [junit4]   2> 3939 DEBUG (AutoscalingActionExecutor-10-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 1091 ms for event 
id=1243a3fe04f3abT15c3qus8f0ezck17grlu6ga28
   [junit4]   2> 3946 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3946 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:            1
   [junit4]   2> 3978 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:   1
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:            1
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##                127.0.0.1:10002_solr
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:   []
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:                0
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##                - autoscaling        2
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##                - update             2
   [junit4]   2> 3979 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 3980 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger
   [junit4]   2> 3981 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##                - STARTED         1
   [junit4]   2> 3981 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##                - SUCCEEDED       1
   [junit4]   2> 3984 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 3984 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 3985 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 3985 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 3985 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 3985 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10002_solr
   [junit4]   2> 3985 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 3985 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 3986 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 3986 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 4076 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 4077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
10
   [junit4]   2> 4077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 4077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 4078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 4079 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4112 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4149 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4174 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4183 DEBUG (simCloudManagerPool-11-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 4184 DEBUG (simCloudManagerPool-11-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 4184 DEBUG (simCloudManagerPool-11-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4184 DEBUG (simCloudManagerPool-11-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4185 DEBUG (simCloudManagerPool-11-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4185 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 4185 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 4185 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 4207 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4207 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 4207 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4208 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 4209 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 4209 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4209 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4209 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4210 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4210 DEBUG (simCloudManagerPool-11-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4210 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 4210 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 4210 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 4211 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 4211 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4228 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 4230 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 4230 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
5140939884097285
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[5140939884097285], now=5140939888247335
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243a8579a6f05T15c3qus8f0ezck17grlu6ga2b",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5140939884097285,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140939884097285],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4247 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger2, node_added_trigger1]
   [junit4]   2> 4249 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243a8579a6f05T15c3qus8f0ezck17grlu6ga2b",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":5140939884097285,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140939884097285],
   [junit4]   2>     "_enqueue_time_":5140939914308835,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4283 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
5140941689851985
   [junit4]   2> 4283 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[5140941689851985], now=5140941699140435
   [junit4]   2> 4283 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243a8c33c1051T15c3qus8f0ezck17grlu6ga2d",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5140941689851985,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140941689851985],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4249 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 4366 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 4366 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 4366 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 4367 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 5869 ms for event 
id=1243a8579a6f05T15c3qus8f0ezck17grlu6ga2b
   [junit4]   2> 4482 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 4482 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 4482 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 4482 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[5140941689851985], now=5140951655385485
   [junit4]   2> 4483 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243a8c33c1051T15c3qus8f0ezck17grlu6ga2g",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5140941689851985,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140941689851985],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4483 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger2, node_added_trigger1]
   [junit4]   2> 4482 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 4484 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 4485 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243a8c33c1051T15c3qus8f0ezck17grlu6ga2g",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":5140941689851985,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140941689851985],
   [junit4]   2>     "_enqueue_time_":5140951679096635,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4485 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration last action at 5140942154443585 time = 
5140951808276885
   [junit4]   2> 4485 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
   [junit4]   2> 4544 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 4544 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 4544 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 4545 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 2973 ms for event 
id=1243a8c33c1051T15c3qus8f0ezck17grlu6ga2g
   [junit4]   2> 4548 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 4548 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
13
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4549 DEBUG (simCloudManagerPool-11-thread-8) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 4550 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 4550 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 4550 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 3
   [junit4]   2> 4596 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 3
   [junit4]   2> 4601 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
14
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4602 DEBUG (simCloudManagerPool-11-thread-9) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 4603 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 4603 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 4603 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 4603 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 3
   [junit4]   2> 4706 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4706 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 2
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10004_solr]
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243adb2b398dfT15c3qus8f0ezck17grlu6ga2j",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":5140962887309535,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140962887309535],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4707 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, 
.auto_add_replicas, node_added_trigger2, node_lost_trigger1, 
node_added_trigger1]
   [junit4]   2> 4709 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 2
   [junit4]   2> 4709 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr
   [junit4]   2> 4709 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10004_solr]
   [junit4]   2> 4709 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243adb8f505b7T15c3qus8f0ezck17grlu6ga2l",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":5140962992260535,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140962992260535],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4724 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243adb2b398dfT15c3qus8f0ezck17grlu6ga2j",
   [junit4]   2>   "source":"node_lost_trigger2",
   [junit4]   2>   "eventTime":5140962887309535,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140962887309535],
   [junit4]   2>     "_enqueue_time_":5140962908389035,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4747 DEBUG (ScheduledTrigger-12-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: 
[127.0.0.1:10004_solr] is not ready, will try later
   [junit4]   2> 4747 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger2
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 4748 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 1308 ms for event 
id=1243adb2b398dfT15c3qus8f0ezck17grlu6ga2j
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with 
currently live nodes: 2
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with 
currently live nodes: 2
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 4849 DEBUG (ScheduledTrigger-12-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 4848 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10004_solr]
   [junit4]   2> 4849 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243adb8f505b7T15c3qus8f0ezck17grlu6ga2o",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":5140962992260535,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140962992260535],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4849 DEBUG (ScheduledTrigger-12-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, 
.auto_add_replicas, node_added_trigger2, node_lost_trigger1, 
node_added_trigger1]
   [junit4]   2> 4850 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243adb8f505b7T15c3qus8f0ezck17grlu6ga2o",
   [junit4]   2>   "source":"node_lost_trigger1",
   [junit4]   2>   "eventTime":5140962992260535,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140962992260535],
   [junit4]   2>     "_enqueue_time_":5140969996927085,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 4850 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration last action at 5140964887877285 time = 
5140970067116785
   [junit4]   2> 4851 INFO  (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger1
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.SolrTestCaseJ4 ###Ending testTriggerThrottling
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:          2
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:          1
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              127.0.0.1:10004_solr
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: []
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:       0
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:       0
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:              0
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 4851 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - autoscaling        4
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - update             9
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - IGNORED         1
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - STARTED         1
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - SUCCEEDED       1
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger2
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - STARTED         1
   [junit4]   2> 4864 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - SUCCEEDED       1
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger1
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - IGNORED         1
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - STARTED         1
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger2
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - STARTED         1
   [junit4]   2> 4865 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[CA6E4E49ADA1E82A]) [   
 ] o.a.s.c.a.s.SimSolrCloudTestCase ##              - SUCCEEDED       1
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
   [junit4]   2> 4909 DEBUG (AutoscalingActionExecutor-13-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 2932 ms for event 
id=1243adb8f505b7T15c3qus8f0ezck17grlu6ga2o
   [junit4]   2> 4923 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 4923 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 15
   [junit4]   2> 4923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 4924 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 4924 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4924 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 4924 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10004_solr
   [junit4]   2> 4924 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 4924 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 4945 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 4953 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 4954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
16
   [junit4]   2> 4954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 4954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion -1
   [junit4]   2> 4954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
   [junit4]   2> 4973 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4973 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 4978 DEBUG (ScheduledTrigger-15-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 5026 DEBUG (ScheduledTrigger-15-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 5047 DEBUG (ScheduledTrigger-15-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 5104 DEBUG (ScheduledTrigger-15-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 5104 DEBUG (simCloudManagerPool-14-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 5105 DEBUG (simCloudManagerPool-14-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
17
   [junit4]   2> 5105 DEBUG (simCloudManagerPool-14-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 5105 DEBUG (simCloudManagerPool-14-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10003_solr]
   [junit4]   2> 5105 DEBUG (simCloudManagerPool-14-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=2, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 5120 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17
   [junit4]   2> 5120 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 5120 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17
   [junit4]   2> 5120 DEBUG (ScheduledTrigger-15-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 5120 DEBUG (ScheduledTrigger-15-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 5121 DEBUG (ScheduledTrigger-15-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 
5140983568529235
   [junit4]   2> 5140 DEBUG (ScheduledTrigger-15-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 5141 DEBUG (ScheduledTrigger-15-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 5141 DEBUG (ScheduledTrigger-15-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 5172 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 5173 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 5173 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 5173 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10005_solr] added at times 
[5140983568529235], now=5140986184934085
   [junit4]   2> 5173 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1243b28365f753T15c3qus8f0ezck17grlu6ga2r",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5140983568529235,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140983568529235],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 5193 DEBUG (ScheduledTrigger-15-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 5229 DEBUG (ScheduledTrigger-15-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
node_added_trigger]
   [junit4]   2> 5250 DEBUG (AutoscalingActionExecutor-16-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1243b28365f753T15c3qus8f0ezck17grlu6ga2r",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":5140983568529235,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[5140983568529235],
   [junit4]   2>     "_enqueue_time_":5140989028372685,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 5267 DEBUG (AutoscalingActionExecutor-16-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 5267 DEBUG (AutoscalingActionExecutor-16-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
   [junit4]   2> 5267 DEBUG (AutoscalingActionExecutor-16-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 833 ms for event 
id=1243b28365f753T15c3qus8f0ezck17grlu6ga2r
   [junit4]   2> 5269 DEBUG (simCloudManagerPool-14-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 5302 DEBUG (simCloudManagerPool-14-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
18
   [junit4]   2> 5303 DEBUG (simCloudManagerPool-14-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10005_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 5303 DEBUG (simCloudManagerPool-14-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10005_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 5303 DEBUG (simCloudManagerPool-14-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=2, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 5319 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18
   [junit4]   2> 5319 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] 
o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 5375 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:           3
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:  3
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:           0
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:  []
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:        0
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:        0
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:               0
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - autoscaling        2
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - update             2
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 5376 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - STARTED         1
   [junit4]   2> 5377 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CA6E4E49ADA1E82A]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##               - SUCCEEDED       1
   [junit4]   2> 5377 DEBUG (ScheduledTrigger-15-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 5378 DEBUG (ScheduledTrigger-15-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4] IGNOR/A 0.00s J0 | TestTriggerIntegration.testEventFromRestoredState
   [junit4]    > Assumption #1: 'badapp

[...truncated too long message...]

essing-n:127.0.0.1:46079_solr) [n:127.0.0.1:46079_solr    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (1)
   [junit4]   2> 531490 INFO  
(zkCallback-266-thread-5-processing-n:127.0.0.1:44458_solr) 
[n:127.0.0.1:44458_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (1)
   [junit4]   2> 531492 INFO  
(zkCallback-262-thread-5-processing-n:127.0.0.1:42806_solr) 
[n:127.0.0.1:42806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (1)
   [junit4]   2> 531492 INFO  
(zkCallback-254-thread-6-processing-n:127.0.0.1:46079_solr) 
[n:127.0.0.1:46079_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (1)
   [junit4]   2> 531510 INFO  (jetty-closer-213-thread-1) [    ] 
o.a.s.c.Overseer Overseer 
(id=72394843383201815-127.0.0.1:35881_solr-n_0000000004) closing
   [junit4]   2> 531524 INFO  
(zkCallback-237-thread-6-processing-n:127.0.0.1:42806_solr) 
[n:127.0.0.1:42806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (1)
   [junit4]   2> 531524 INFO  (jetty-closer-213-thread-2) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:42806_solr
   [junit4]   2> 531525 WARN  
(OverseerAutoScalingTriggerThread-72394843383201815-127.0.0.1:35881_solr-n_0000000004)
 [n:127.0.0.1:35881_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 531528 INFO  
(OverseerStateUpdate-72394843383201815-127.0.0.1:35881_solr-n_0000000004) 
[n:127.0.0.1:35881_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:35881_solr
   [junit4]   2> 531529 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x10132b9fb6d0019, likely client has closed socket
   [junit4]   2> 531578 INFO  (jetty-closer-213-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 531578 INFO  
(zkCallback-258-thread-5-processing-n:127.0.0.1:35881_solr) 
[n:127.0.0.1:35881_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 531668 INFO  
(zkCallback-239-thread-6-processing-n:127.0.0.1:35881_solr) 
[n:127.0.0.1:35881_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 531668 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x10132b9fb6d001b, likely client has closed socket
   [junit4]   2> 531668 INFO  
(zkCallback-237-thread-7-processing-n:127.0.0.1:42806_solr) 
[n:127.0.0.1:42806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 531668 INFO  
(zkCallback-262-thread-5-processing-n:127.0.0.1:42806_solr) 
[n:127.0.0.1:42806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 531695 WARN  
(zkCallback-246-thread-7-processing-n:127.0.0.1:44458_solr) 
[n:127.0.0.1:44458_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 531698 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x10132b9fb6d0015, likely client has closed socket
   [junit4]   2> 531723 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 
0x10132b9fb6d0017, likely client has closed socket
   [junit4]   2> 531728 INFO  (jetty-closer-213-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@14f650{/solr,null,UNAVAILABLE}
   [junit4]   2> 531728 INFO  (jetty-closer-213-thread-4) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@531b899d{/solr,null,UNAVAILABLE}
   [junit4]   2> 531729 INFO  (jetty-closer-213-thread-1) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 531729 INFO  
(zkCallback-237-thread-8-processing-n:127.0.0.1:42806_solr) 
[n:127.0.0.1:42806_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:42806_solr
   [junit4]   2> 531729 INFO  (jetty-closer-213-thread-4) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 533181 WARN  
(zkCallback-241-thread-9-processing-n:127.0.0.1:46079_solr) 
[n:127.0.0.1:46079_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 533202 INFO  (jetty-closer-213-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@6afe2b9c{/solr,null,UNAVAILABLE}
   [junit4]   2> 533202 INFO  (jetty-closer-213-thread-3) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 533253 INFO  (jetty-closer-213-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@76219de5{/solr,null,UNAVAILABLE}
   [junit4]   2> 533254 INFO  (jetty-closer-213-thread-2) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 533323 ERROR 
(SUITE-JDBCStreamTest-seed#[D9AF7EB640850D97]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 533328 INFO  
(SUITE-JDBCStreamTest-seed#[D9AF7EB640850D97]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:32955 32955
   [junit4]   2> 533430 INFO  (Thread-177) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:32955 32955
   [junit4]   2> 533430 WARN  (Thread-177) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        18      /solr/aliases.json
   [junit4]   2>        18      /solr/clusterprops.json
   [junit4]   2>        8       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        18      /solr/clusterstate.json
   [junit4]   2>        8       /solr/collections/jdbc/state.json
   [junit4]   2>        2       /solr/autoscaling.json
   [junit4]   2>        2       
/solr/overseer_elect/election/72394843383201815-127.0.0.1:35881_solr-n_0000000004
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        18      /solr/live_nodes
   [junit4]   2>        18      /solr/collections
   [junit4]   2>        2       /solr/overseer/queue
   [junit4]   2>        2       /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2>        2       /solr/overseer/collection-queue-work
   [junit4]   2>        2       /solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 533441 INFO  
(SUITE-JDBCStreamTest-seed#[D9AF7EB640850D97]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ------------------------------------------------------- 
Done waiting for tracked resources to be released
   [junit4]   2> Apr 04, 2018 2:26:44 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=410, maxMBSortInHeap=5.563828939043081, 
sim=RandomSimilarity(queryNorm=false): {}, locale=es-VE, timezone=Europe/Dublin
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=156238096,total=381157376
   [junit4]   2> NOTE: All tests run in this JVM: [CoalesceEvaluatorTest, 
JdbcDriverTest, SolrExceptionTest, AddEvaluatorTest, CosineEvaluatorTest, 
MultiplyEvaluatorTest, SolrExampleBinaryTest, PowerEvaluatorTest, 
TestSolrJErrorHandling, TestJavaBinCodec, TestHash, SolrParamTest, 
ArcCosineEvaluatorTest, TestSuggesterResponse, RegressionEvaluatorTest, 
SolrExampleXMLTest, CommonParamsTest, TestCloudSolrClientConnections, 
JettyWebappTest, TestValidatingJsonMap, CloudSolrClientTest, 
CloudSolrClientBuilderTest, LessThanEqualToEvaluatorTest, 
SelectWithEvaluatorsTest, TestCollectionAdminRequest, JDBCStreamTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=JDBCStreamTest 
-Dtests.seed=D9AF7EB640850D97 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=es-VE -Dtests.timezone=Europe/Dublin -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J2 | JDBCStreamTest (suite) <<<
   [junit4]    > Throwable #1: 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:46079/solr: KeeperErrorCode = NoNode for 
/overseer/collection-queue-work/qnr-0000000000
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([D9AF7EB640850D97]:0)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
   [junit4]    >        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    >        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    >        at 
org.apache.solr.client.solrj.io.stream.JDBCStreamTest.setupCluster(JDBCStreamTest.java:76)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [145/146 (2!)] on J2 in 192.30s, 0 tests, 1 error <<< 
FAILURES!

[...truncated 48643 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to