[JENKINS] Lucene-Solr-Tests-master - Build # 2260 - Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2260/

4 tests failed.
FAILED:  
org.apache.solr.cloud.TestLeaderInitiatedRecoveryThread.testPublishDownState

Error Message:
expected:<27> but was:<28>

Stack Trace:
java.lang.AssertionError: expected:<27> but was:<28>
        at 
__randomizedtesting.SeedInfo.seed([DE3B19B2F101D8D9:8046BB4C551C9124]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.TestLeaderInitiatedRecoveryThread.testPublishDownState(TestLeaderInitiatedRecoveryThread.java:115)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToPullReplica

Error Message:
Timeout occured while waiting response from server at: 
http://127.0.0.1:33568/solr/pull_replica_error_handling_test_cant_connect_to_pull_replica_shard1_replica_n1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: 
http://127.0.0.1:33568/solr/pull_replica_error_handling_test_cant_connect_to_pull_replica_shard1_replica_n1
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at 
org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:241)
        at 
org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:251)
        at 
org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToPullReplica(TestPullReplicaErrorHandling.java:151)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 47 more


FAILED:  
org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple

Error Message:
No live SolrServers available to handle this 
request:[http://127.0.0.1:33259/solr]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available 
to handle this request:[http://127.0.0.1:33259/solr]
        at 
__randomizedtesting.SeedInfo.seed([DE3B19B2F101D8D9:E6883D4CD6F20C08]:0)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
        at 
org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple(AutoAddReplicasPlanActionTest.java:110)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException 
occured when talking to server at: http://127.0.0.1:33259/solr
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        ... 44 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:33259 failed to 
respond
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
        ... 48 more


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode

Error Message:
did not finish processing changes

Stack Trace:
java.lang.AssertionError: did not finish processing changes
        at 
__randomizedtesting.SeedInfo.seed([DE3B19B2F101D8D9:79D404113E4C57C1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:301)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11761 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestLargeCluster_DE3B19B2F101D8D9-001/init-core-data-001
   [junit4]   2> 280040 WARN  
(SUITE-TestLargeCluster-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=35 numCloses=35
   [junit4]   2> 280040 INFO  
(SUITE-TestLargeCluster-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 280041 INFO  
(SUITE-TestLargeCluster-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 280042 INFO  
(SUITE-TestLargeCluster-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 280205 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 280242 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 280242 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 280243 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 280243 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 280244 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 280244 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 280245 DEBUG (ScheduledTrigger-402-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280271 INFO  
(TEST-TestLargeCluster.testAddNode-seed#[DE3B19B2F101D8D9]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testAddNode
   [junit4]   2> 280271 DEBUG 
(TEST-TestLargeCluster.testAddNode-seed#[DE3B19B2F101D8D9]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 280274 DEBUG (ScheduledTrigger-402-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280290 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 280291 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 280292 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 280292 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 280292 INFO  
(TEST-TestLargeCluster.testAddNode-seed#[DE3B19B2F101D8D9]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 280293 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 280293 DEBUG 
(TEST-TestLargeCluster.testAddNode-seed#[DE3B19B2F101D8D9]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 280294 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 280343 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280367 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280388 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280420 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280477 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280497 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280529 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280550 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280570 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280620 DEBUG (simCloudManagerPool-404-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 280620 DEBUG (simCloudManagerPool-404-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 280621 DEBUG (simCloudManagerPool-404-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 280623 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280644 DEBUG (simCloudManagerPool-404-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 280644 DEBUG (simCloudManagerPool-404-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=5, actions=[{name=compute, 
class=org.apache.solr.cloud.autoscaling.ComputePlanAction}, {name=execute, 
class=org.apache.solr.cloud.autoscaling.ExecutePlanAction}], enabled=true}
   [junit4]   2> 280644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 280645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 280659 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280660 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280660 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280680 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280680 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280695 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280716 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280732 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280732 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280752 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280752 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280752 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280825 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280825 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280825 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280864 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280864 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280864 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280899 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280899 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280899 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280934 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280935 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280935 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 280971 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 280971 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 280971 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281007 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281008 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281008 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281044 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281044 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281044 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281080 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281080 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281081 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281117 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281117 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281117 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281153 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281153 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281153 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281190 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281190 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281190 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281242 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281243 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281243 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281263 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281263 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281263 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281299 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281299 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281299 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281336 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281336 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281336 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281372 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281372 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281372 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281409 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281409 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281409 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281445 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281445 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281445 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281481 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281482 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281482 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281518 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281518 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281518 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281554 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281554 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281554 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281591 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281591 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281591 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281664 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281664 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281664 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281684 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281684 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281684 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281704 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281704 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281704 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281725 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281725 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281725 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281745 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281745 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281745 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281765 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281765 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281765 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281789 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281814 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281814 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281814 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281850 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281850 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281850 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281886 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281887 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281887 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281907 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281907 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281907 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281927 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281927 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281927 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281955 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281956 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281956 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 281992 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 281992 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 281992 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282028 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282028 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282028 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282065 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282065 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282065 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282101 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282101 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282101 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282138 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282138 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282138 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282174 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282174 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282174 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282211 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282211 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282211 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282247 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282247 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282247 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282284 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282284 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282284 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282320 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282320 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282320 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282356 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282356 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282357 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282393 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282393 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282393 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282429 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282429 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282429 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282450 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282450 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282450 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282482 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282482 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282482 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282518 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282518 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282519 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282539 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282539 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282539 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282559 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282559 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282559 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282579 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282579 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282579 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282614 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282614 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282614 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282634 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282635 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282635 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282689 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282689 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282689 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282709 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282709 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282709 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282729 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282729 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282729 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282761 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282762 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282762 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282790 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282790 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282790 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282810 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282810 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282810 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282830 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282831 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282831 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282851 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282851 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282851 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282871 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282871 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282871 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282891 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282891 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282891 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282912 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282912 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282912 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282932 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282932 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282932 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282964 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 282964 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282964 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282996 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 282996 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 282997 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283017 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283017 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283017 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283049 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283049 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283049 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283069 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283069 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283069 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283106 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283106 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283106 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283142 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283142 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283142 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283180 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283180 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283180 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283215 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283216 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283216 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283236 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283236 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283236 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283272 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283272 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283272 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283308 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283308 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283308 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283357 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283357 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283357 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283389 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283389 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283389 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283410 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283410 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283410 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283438 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283438 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283438 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283458 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283458 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283458 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283478 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283478 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283479 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283511 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283511 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283511 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283531 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283531 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283531 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283551 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283551 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283551 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283588 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283588 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283588 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283616 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283616 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283616 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283653 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283653 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283653 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283677 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283677 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283677 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283705 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283705 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283705 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283726 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283726 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283726 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283746 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283746 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283746 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283778 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283778 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283778 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283815 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283815 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283815 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283835 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283835 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283835 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283855 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283855 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283855 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283888 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283888 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283888 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283908 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283908 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283908 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283940 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283940 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283940 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283960 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283961 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 283961 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283981 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 283981 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 283981 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284001 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284001 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284001 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284021 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284021 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284021 DEBUG (ScheduledTrigger-405-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284042 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284042 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284042 DEBUG (ScheduledTrigger-405-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284063 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284063 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284063 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284083 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284083 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284083 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284103 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 284104 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284104 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284147 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 284147 DEBUG (ScheduledTrigger-405-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 284147 DEBUG (ScheduledTrigger-405-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .aut

[...truncated too long message...]

ng_test_cant_connect_to_pull_replica/state.json] for collection 
[pull_replica_error_handling_test_cant_connect_to_pull_replica] has occurred - 
updating... (live nodes size: [0])
   [junit4]   2> 4589628 INFO  
(zkCallback-3789-thread-10-processing-n:127.0.0.1:33568_solr) 
[n:127.0.0.1:33568_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDeleted 
path:/collections/pull_replica_error_handling_test_cant_connect_to_pull_replica/state.json]
 for collection [pull_replica_error_handling_test_cant_connect_to_pull_replica] 
has occurred - updating... (live nodes size: [0])
   [junit4]   2> 4589630 ERROR 
(OverseerThreadFactory-7582-thread-4-processing-n:127.0.0.1:33568_solr) 
[n:127.0.0.1:33568_solr    ] o.a.s.c.a.c.OverseerCollectionMessageHandler 
Collection: pull_replica_error_handling_test_cant_connect_to_pull_replica 
operation: delete failed:org.apache.solr.common.SolrException: Could not fully 
remove collection: pull_replica_error_handling_test_cant_connect_to_pull_replica
   [junit4]   2>        at 
org.apache.solr.cloud.api.collections.DeleteCollectionCmd.call(DeleteCollectionCmd.java:120)
   [junit4]   2>        at 
org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:251)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:469)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 4589631 INFO  
(OverseerThreadFactory-7582-thread-4-processing-n:127.0.0.1:33568_solr) 
[n:127.0.0.1:33568_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 4589643 WARN  
(OverseerAutoScalingTriggerThread-99362947294232593-127.0.0.1:33568_solr-n_0000000004)
 [n:127.0.0.1:33568_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 4589644 INFO  (jetty-closer-3734-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@31da24c0{/solr,null,UNAVAILABLE}
   [junit4]   2> 4589644 INFO  (jetty-closer-3734-thread-1) [    ] 
o.e.j.s.session Stopped scavenging
   [junit4]   2> 4589645 ERROR 
(SUITE-TestPullReplicaErrorHandling-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4589645 INFO  
(SUITE-TestPullReplicaErrorHandling-seed#[DE3B19B2F101D8D9]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:37063 37063
   [junit4]   2> 4589769 INFO  (Thread-4212) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:37063 37063
   [junit4]   2> 4589769 WARN  (Thread-4212) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        10      /solr/aliases.json
   [junit4]   2>        9       /solr/security.json
   [junit4]   2>        9       /solr/configs/conf
   [junit4]   2>        9       /solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        28      
/solr/collections/pull_replica_error_handling_test_cant_connect_to_pull_replica/state.json
   [junit4]   2>        19      
/solr/collections/pull_replica_error_handling_test_pull_replica_disconnects_from_zoo_keeper/state.json
   [junit4]   2>        10      
/solr/collections/pull_replica_error_handling_test_cant_connect_to_leader/state.json
   [junit4]   2>        10      /solr/clusterstate.json
   [junit4]   2>        6       /solr/clusterprops.json
   [junit4]   2>        3       /solr/autoscaling.json
   [junit4]   2>        3       
/solr/overseer_elect/election/99362947294232593-127.0.0.1:33568_solr-n_0000000004
   [junit4]   2>        2       
/solr/overseer_elect/election/99362947294232599-127.0.0.1:43947_solr-n_0000000006
   [junit4]   2>        2       
/solr/overseer_elect/election/99362947294232585-127.0.0.1:46158_solr-n_0000000000
   [junit4]   2>        2       
/solr/overseer_elect/election/99362947294232588-127.0.0.1:38468_solr-n_0000000002
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        10      /solr/collections
   [junit4]   2>        9       /solr/live_nodes
   [junit4]   2>        3       /solr/overseer/queue
   [junit4]   2>        3       /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2>        3       /solr/overseer/queue-work
   [junit4]   2>        3       /solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplicaErrorHandling_DE3B19B2F101D8D9-001
   [junit4]   2> Jan 17, 2018 3:22:41 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: 
codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST,
 chunkSize=3, maxDocsPerChunk=587, blockSize=830), 
termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, 
chunkSize=3, blockSize=830)), 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3386c1aa),
 locale=ca, timezone=America/Dominica
   [junit4]   2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=128602880,total=520093696
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreBackup, 
LeaderElectionIntegrationTest, MoveReplicaHDFSFailoverTest, 
StatelessScriptUpdateProcessorFactoryTest, StatsReloadRaceTest, TestFaceting, 
PingRequestHandlerTest, DocValuesNotIndexedTest, TestConfigsApi, 
TestJavabinTupleStreamParser, ZkStateWriterTest, TestAnalyzedSuggestions, 
TestPrepRecovery, DirectSolrSpellCheckerTest, ReturnFieldsTest, 
TestRandomFlRTGCloud, TestLFUCache, TestGenericDistributedQueue, 
CollectionsAPISolrJTest, TestReqParamsAPI, PreAnalyzedFieldTest, 
TestDocBasedVersionConstraints, InfixSuggestersTest, SearchHandlerTest, 
OverseerRolesTest, TestExportWriter, AnalysisAfterCoreReloadTest, 
TestNumericRangeQuery64, SolrTestCaseJ4Test, TestPolicyCloud, 
UniqFieldsUpdateProcessorFactoryTest, DistributedFacetPivotLargeTest, 
TestConfigSetsAPIZkFailure, JvmMetricsTest, ResponseHeaderTest, 
TestAtomicUpdateErrorCases, SpatialFilterTest, CurrencyFieldTypeTest, 
TestUtils, TestQuerySenderListener, SolrCmdDistributorTest, GraphQueryTest, 
RecoveryZkTest, OverriddenZkACLAndCredentialsProvidersTest, 
TestLocalFSCloudBackupRestore, TestManagedStopFilterFactory, 
LegacyCloudClusterPropTest, TestLegacyTerms, CreateCollectionCleanupTest, 
ReplaceNodeTest, TestComplexPhraseQParserPlugin, TestRestManager, 
DisMaxRequestHandlerTest, ClusterStateTest, SpatialHeatmapFacetsTest, 
TestInitQParser, HdfsNNFailoverTest, TestSuggestSpellingConverter, 
SpellPossibilityIteratorTest, HLLUtilTest, TestPivotHelperCode, 
MultiThreadedOCPTest, AtomicUpdatesTest, NumericFieldsTest, NoCacheHeaderTest, 
NodeLostTriggerTest, LukeRequestHandlerTest, SmileWriterTest, 
VMParamsZkACLAndCredentialsProvidersTest, TestAuthorizationFramework, 
TestNodeLostTrigger, TestRangeQuery, UnloadDistributedZkTest, 
TestSolrCloudWithDelegationTokens, CurrencyRangeFacetCloudTest, StressHdfsTest, 
TestExactSharedStatsCache, BasicAuthIntegrationTest, 
CachingDirectoryFactoryTest, TestRequestForwarding, DocValuesMultiTest, 
TestNRTOpen, TestStandardQParsers, SolrJmxReporterTest, 
TestSolrDeletionPolicy2, MinimalSchemaTest, SharedFSAutoReplicaFailoverTest, 
SolrXmlInZkTest, TestTolerantSearch, ClusterStateUpdateTest, 
TestObjectReleaseTracker, TestDFISimilarityFactory, ComputePlanActionTest, 
TestMissingGroups, TestRandomDVFaceting, CloudMLTQParserTest, 
SolrGangliaReporterTest, DateFieldTest, TestLeaderElectionWithEmptyReplica, 
TestLazyCores, LeaderFailureAfterFreshStartTest, TestCollationFieldDocValues, 
TestCoreContainer, TestDistributedStatsComponentCardinality, TestTlogReplica, 
TestQueryTypes, DeleteNodeTest, ConvertedLegacyTest, MultiTermTest, 
TestComputePlanAction, TestSolr4Spatial2, SolrShardReporterTest, 
TestShortCircuitedRequests, RulesTest, SearchHandlerTest, 
TestSchemaNameResource, CdcrBootstrapTest, RegexBoostProcessorTest, 
TestLegacyField, TestConfigSetImmutable, TestSchemaManager, 
CoreMergeIndexesAdminHandlerTest, TestSystemCollAutoCreate, TimeZoneUtilsTest, 
TestJsonFacetRefinement, TestRecovery, ZkControllerTest, TestRealTimeGet, 
TestJoin, TestStressVersions, SpellCheckComponentTest, BadIndexSchemaTest, 
TestSort, DirectUpdateHandlerTest, TestIndexSearcher, 
DistributedQueryElevationComponentTest, DocValuesTest, MoreLikeThisHandlerTest, 
FastVectorHighlighterTest, SolrPluginUtilsTest, QueryParsingTest, 
TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, 
AlternateDirectoryTest, CopyFieldTest, TestBinaryField, TestSearchPerf, 
ExternalFileFieldSortTest, TestSolrCoreProperties, NotRequiredUniqueKeyTest, 
EchoParamsTest, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, 
URLClassifyProcessorTest, PluginInfoTest, DOMUtilTest, TestSolrJ, 
TestDocumentBuilder, UUIDFieldTest, CircularListTest, CursorPagingTest, 
TestLuceneIndexBackCompat, AliasIntegrationTest, CleanupOldIndexTest, 
DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, ForceLeaderTest, 
PeerSyncReplicationTest, SSLMigrationTest, SaslZkACLProviderTest, 
TestCloudInspectUtil, TestCloudPseudoReturnFields, TestClusterProperties, 
TestConfigSetsAPI, TestLeaderInitiatedRecoveryThread, 
TestPullReplicaErrorHandling]
   [junit4] Completed [756/768 (4!)] on J2 in 1108.64s, 3 tests, 1 error <<< 
FAILURES!

[...truncated 48087 lines...]

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org