Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/3591/
Java: 64bit/jdk-12-ea+shipilev-fastdebug -XX:+UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED:
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testEventQueue
Error Message:
action wasn't interrupted
Stack Trace:
java.lang.AssertionError: action wasn't interrupted
at
__randomizedtesting.SeedInfo.seed([A2EF65D9003E0910:6B5A27770959CFE5]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testEventQueue(TestSimTriggerIntegration.java:757)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:835)
FAILED:
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testSearchRate
Error Message:
last ClusterState: znodeVersion: 6 live nodes:[127.0.0.1:10019_solr,
127.0.0.1:10020_solr]
collections:{collection1=DocCollection(collection1//clusterstate.json/5)={
"replicationFactor":"1", "pullReplicas":"0",
"router":{"name":"compositeId"}, "maxShardsPerNode":"1",
"autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0",
"autoCreated":"true", "shards":{"shard1":{ "replicas":{
"core_node1":{ "core":"collection1_shard1_replica_n1",
"SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10020_solr",
"state":"active", "type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0},
"core_node2":{ "core":"collection1_shard1_replica_n2",
"SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10019_solr",
"state":"active", "type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff", "state":"active"}}}}, last coll state:
DocCollection(collection1//clusterstate.json/5)={ "replicationFactor":"1",
"pullReplicas":"0", "router":{"name":"compositeId"},
"maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2",
"tlogReplicas":"0", "autoCreated":"true", "shards":{"shard1":{
"replicas":{ "core_node1":{
"core":"collection1_shard1_replica_n1", "SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10020_solr",
"state":"active", "type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0},
"core_node2":{ "core":"collection1_shard1_replica_n2",
"SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10019_solr",
"state":"active", "type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff", "state":"active"}}}
Stack Trace:
java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 6
live nodes:[127.0.0.1:10019_solr, 127.0.0.1:10020_solr]
collections:{collection1=DocCollection(collection1//clusterstate.json/5)={
"replicationFactor":"1",
"pullReplicas":"0",
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0",
"autoCreated":"true",
"shards":{"shard1":{
"replicas":{
"core_node1":{
"core":"collection1_shard1_replica_n1",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10020_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node2":{
"core":"collection1_shard1_replica_n2",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10019_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff",
"state":"active"}}}}, last coll state:
DocCollection(collection1//clusterstate.json/5)={
"replicationFactor":"1",
"pullReplicas":"0",
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0",
"autoCreated":"true",
"shards":{"shard1":{
"replicas":{
"core_node1":{
"core":"collection1_shard1_replica_n1",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10020_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node2":{
"core":"collection1_shard1_replica_n2",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10019_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff",
"state":"active"}}}
at
__randomizedtesting.SeedInfo.seed([A2EF65D9003E0910:FFA77B50CFF8AF5F]:0)
at
org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:129)
at
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testSearchRate(TestSimTriggerIntegration.java:1330)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:835)
Build Log:
[...truncated 12922 lines...]
[junit4] Suite:
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[junit4] 2> Creating dataDir:
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestSimTriggerIntegration_A2EF65D9003E0910-001/init-core-data-001
[junit4] 2> 304033 INFO
(SUITE-TestSimTriggerIntegration-seed#[A2EF65D9003E0910]-worker) [ ]
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 304034 INFO
(SUITE-TestSimTriggerIntegration-seed#[A2EF65D9003E0910]-worker) [ ]
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via:
@org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0,
clientAuth=0.0/0.0)
[junit4] 2> 304034 INFO
(SUITE-TestSimTriggerIntegration-seed#[A2EF65D9003E0910]-worker) [ ]
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 304037 INFO
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.SolrTestCaseJ4 ###Starting testCooldown
[junit4] 2> 304253 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 304254 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 1
[junit4] 2> 304267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 304267 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader:
127.0.0.1:10000_solr
[junit4] 2> 304267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 304268 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states,
currentVersion=0
[junit4] 2> 304302 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 304303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 304303 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 304374 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 1 and last live nodes: 1
[junit4] 2> 304392 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 304392 INFO
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history
in memory.
[junit4] 2> 304416 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 1
[junit4] 2> 304437 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304457 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304477 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304498 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304518 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304539 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304559 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304579 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304603 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304623 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304643 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304664 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304684 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304705 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304725 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304746 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304766 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304786 DEBUG (simCloudManagerPool-814-thread-2) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 304786 DEBUG (simCloudManagerPool-814-thread-2) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
2
[junit4] 2> 304787 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304787 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 304788 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 304788 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 304807 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304827 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304848 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304868 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304889 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304909 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304930 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304947 DEBUG (simCloudManagerPool-814-thread-3) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 304947 DEBUG (simCloudManagerPool-814-thread-3) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
3
[junit4] 2> 304950 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304971 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 304991 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305011 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305032 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305036 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 305037 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 305037 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger -
Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 305037 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test,
class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}],
enabled=true}
[junit4] 2> 305052 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305058 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305058 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 305072 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305076 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 305076 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 305093 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305104 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305104 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 305108 DEBUG (simCloudManagerPool-814-thread-4) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 305109 DEBUG (simCloudManagerPool-814-thread-4) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
4
[junit4] 2> 305109 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 305110 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 305110 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 305113 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305124 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305124 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 305133 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305144 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305145 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 305154 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 305165 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305165 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 305172 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- creating marker:
/autoscaling/nodeAdded/127.0.0.1:10002_solr
[junit4] 2> 305174 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 2
[junit4] 2> 305186 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305186 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 305186 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time
982636795600352
[junit4] 2> 305195 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305207 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305207 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 305207 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing
registered processor for nodes: [127.0.0.1:10002_solr] added at times
[982636795600352], now=982637832232952
[junit4] 2> 305215 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305236 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305256 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305276 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305297 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305317 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 305322 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"37db3f5251de0T7j5gt4o1b2my72udkkgss66hb",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982636795600352,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982636795600352],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 305323 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
node_added_cooldown_trigger]
[junit4] 2> 305342 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37db3f5251de0T7j5gt4o1b2my72udkkgss66hb",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982636795600352,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982636795600352],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982644574451252,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 305544 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"37db3f5251de0T7j5gt4o1b2my72udkkgss66hb",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982636795600352,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982636795600352],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982644574451252,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 305561 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10002_solr
[junit4] 2> 305677 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37db3f5251de0T7j5gt4o1b2my72udkkgss66hb",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982636795600352,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982636795600352],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982644574451252,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 305715 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- creating marker:
/autoscaling/nodeAdded/127.0.0.1:10003_solr
[junit4] 2> 305775 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 305775 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_cooldown_trigger after
100ms
[junit4] 2> 305775 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 11578 ms for event
id=37db3f5251de0T7j5gt4o1b2my72udkkgss66hb
[junit4] 2> 305876 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305876 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 3
[junit4] 2> 305876 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 305876 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time
982671282671702
[junit4] 2> 305903 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing
registered processor for nodes: [127.0.0.1:10003_solr] added at times
[982671282671702], now=982672680026502
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"37dbbfcbc1056T7j5gt4o1b2my72udkkgss66hc",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982671282671702,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982671282671702],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
node_added_cooldown_trigger]
[junit4] 2> 305904 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37dbbfcbc1056T7j5gt4o1b2my72udkkgss66hc",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982671282671702,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982671282671702],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982672703420402,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 305905 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10003_solr
[junit4] 2> 305905 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"37dbbfcbc1056T7j5gt4o1b2my72udkkgss66hc",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982671282671702,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982671282671702],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982672703420402,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 305905 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37dbbfcbc1056T7j5gt4o1b2my72udkkgss66hc",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":982671282671702,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982671282671702],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982672703420402,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 305905 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_cooldown_trigger after
100ms
[junit4] 2> 305905 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 305905 DEBUG (AutoscalingActionExecutor-816-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 26 ms for event
id=37dbbfcbc1056T7j5gt4o1b2my72udkkgss66hc
[junit4] 2> 306005 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306005 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306006 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306026 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306026 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306026 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306046 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306046 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306046 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306066 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306066 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306066 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306087 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306087 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306087 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306107 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306107 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306107 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306127 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306127 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306127 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306147 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306148 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306148 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306168 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306168 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306168 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306188 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306188 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306188 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306208 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306209 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306209 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306229 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306229 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306229 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306249 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306268 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306268 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306269 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306288 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306288 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306290 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306308 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306308 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306310 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306329 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306329 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306330 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306349 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306349 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306350 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306369 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306370 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306371 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306390 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306390 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306391 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306410 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306410 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306411 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306431 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306431 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306431 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306451 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306452 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306459 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306472 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306479 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306479 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306492 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306499 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306500 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306513 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306520 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306520 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306533 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306540 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306540 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306553 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306561 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306561 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306574 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306581 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306581 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306594 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306601 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306602 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306614 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306622 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306622 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306635 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306642 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306642 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306657 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306663 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306663 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306678 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306683 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306683 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306698 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306703 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306703 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306726 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306726 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306726 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306746 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306747 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306747 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306767 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306767 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306767 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306787 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306787 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306787 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306807 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306807 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306807 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306829 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306829 DEBUG (ScheduledTrigger-815-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306829 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306849 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306849 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306849 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306869 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306869 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306869 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306889 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306890 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306890 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306910 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306910 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306910 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306930 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306930 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306930 DEBUG (ScheduledTrigger-815-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306951 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306952 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306952 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306972 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306972 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306972 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 306993 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 306993 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 306993 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307013 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307013 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307013 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307033 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307033 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307033 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307053 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307053 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307053 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307074 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307074 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307074 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307094 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307094 DEBUG (ScheduledTrigger-815-thread-1) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307094 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307114 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 4 and last live nodes: 4
[junit4] 2> 307114 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 307114 DEBUG (ScheduledTrigger-815-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 307134 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 307134 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exiting.
[junit4] 2> 307135 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 307136 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 307136 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool
executor
[junit4] 2> 307136 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 307136 DEBUG
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 307137 INFO
(TEST-TestSimTriggerIntegration.testCooldown-seed#[A2EF65D9003E0910]) [ ]
o.a.s.SolrTestCaseJ4 ###Ending testCooldown
[junit4] 2> 307140 INFO
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
[junit4] 2> 307148 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader:
127.0.0.1:10004_solr
[junit4] 2> 307148 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 307149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 1
[junit4] 2> 307149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 307150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 307151 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states,
currentVersion=0
[junit4] 2> 307152 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 307152 INFO
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics
history in memory.
[junit4] 2> 307156 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 307168 DEBUG (ScheduledTrigger-821-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307172 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307172 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 307181 DEBUG (simCloudManagerPool-820-thread-2) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307181 DEBUG (simCloudManagerPool-820-thread-2) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
2
[junit4] 2> 307182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 307182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 307188 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307189 DEBUG (simCloudManagerPool-820-thread-3) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307189 DEBUG (simCloudManagerPool-820-thread-3) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
3
[junit4] 2> 307189 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 307190 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 307190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger -
Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr]
[junit4] 2> 307190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger
instantiated with properties: {event=nodeAdded, waitFor=500000,
actions=[{name=test,
class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}],
enabled=true}
[junit4] 2> 307190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307190 DEBUG (ScheduledTrigger-821-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 307190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 307190 DEBUG (ScheduledTrigger-821-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 307195 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker:
/autoscaling/nodeAdded/127.0.0.1:10006_solr
[junit4] 2> 307208 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 2
[junit4] 2> 307210 DEBUG (ScheduledTrigger-821-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 307211 DEBUG (ScheduledTrigger-821-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 307211 DEBUG (ScheduledTrigger-821-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10006_solr at time
982738021470102
[junit4] 2> 307230 DEBUG (ScheduledTrigger-821-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 307231 DEBUG (ScheduledTrigger-821-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 307231 DEBUG (ScheduledTrigger-821-thread-4) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 307232 DEBUG (simCloudManagerPool-820-thread-4) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307232 DEBUG (simCloudManagerPool-820-thread-4) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
4
[junit4] 2> 307233 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 307233 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 307233 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger -
Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10006_solr,
127.0.0.1:10005_solr]
[junit4] 2> 307233 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger
instantiated with properties: {event=nodeAdded, waitFor=2, actions=[{name=test,
class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}],
enabled=true}
[junit4] 2> 307233 DEBUG (ScheduledTrigger-821-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 307233 DEBUG (ScheduledTrigger-821-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 307233 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307234 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 307251 DEBUG (ScheduledTrigger-821-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 307254 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 307254 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 307254 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing
registered processor for nodes: [127.0.0.1:10006_solr] added at times
[982738021470102], now=982740200504952
[junit4] 2> 307254 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"37dcb86ad4f96T7j5gt4o1b2my72udkkgss66he",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":982738021470102,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982738021470102],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}}
[junit4] 2> 307254 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
node_added_restore_trigger]
[junit4] 2> 307255 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37dcb86ad4f96T7j5gt4o1b2my72udkkgss66he",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":982738021470102,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982738021470102],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982740215928802,
[junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}}
[junit4] 2> 307255 DEBUG (ScheduledTrigger-821-thread-2) [ ]
o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10006_solr
[junit4] 2> 307256 DEBUG (AutoscalingActionExecutor-822-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"37dcb86ad4f96T7j5gt4o1b2my72udkkgss66he",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":982738021470102,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982738021470102],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982740215928802,
[junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}}
[junit4] 2> 307257 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool
executor now
[junit4] 2> 307258 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 307258 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 307258 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exiting.
[junit4] 2> 307259 WARN (AutoscalingActionExecutor-822-thread-1) [ ]
o.a.s.c.a.SystemLogListener Exception sending event. Collection: .system,
event: {
[junit4] 2> "id":"37dcb86ad4f96T7j5gt4o1b2my72udkkgss66he",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":982738021470102,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982738021470102],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982740215928802,
[junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}}, exception:
java.io.IOException: already closed
[junit4] 2> 307259 DEBUG (AutoscalingActionExecutor-822-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 175 ms for event
id=37dcb86ad4f96T7j5gt4o1b2my72udkkgss66he
[junit4] 2> 307259 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread
pool executor
[junit4] 2> 307259 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 307259 DEBUG
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed
explicitly
[junit4] 2> 307260 INFO
(TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
[junit4] 2> 307265 INFO
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
[junit4] 2> 307268 DEBUG
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader:
127.0.0.1:10007_solr
[junit4] 2> 307269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 307269 DEBUG
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states,
currentVersion=0
[junit4] 2> 307269 DEBUG
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 307269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 1
[junit4] 2> 307269 INFO
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics
history in memory.
[junit4] 2> 307269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 307270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 307279 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10008_solr, 127.0.0.1:10007_solr]
[junit4] 2> 307284 DEBUG (simCloudManagerPool-826-thread-2) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307284 DEBUG (simCloudManagerPool-826-thread-2) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
2
[junit4] 2> 307285 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307353 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307355 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 307360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 307360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 307365 DEBUG (simCloudManagerPool-826-thread-3) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307365 DEBUG (simCloudManagerPool-826-thread-3) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
3
[junit4] 2> 307365 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 307366 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 307366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial
livenodes: [127.0.0.1:10008_solr, 127.0.0.1:10007_solr]
[junit4] 2> 307366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307366 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 307375 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 3 and last live nodes: 2
[junit4] 2> 307386 DEBUG (ScheduledTrigger-827-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 307405 DEBUG (ScheduledTrigger-827-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 3
[junit4] 2> 307405 DEBUG (ScheduledTrigger-827-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10009_solr
[junit4] 2> 307406 DEBUG
(TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker:
/autoscaling/nodeLost/127.0.0.1:10009_solr
[junit4] 2> 307407 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 2 and last live nodes: 3
[junit4] 2> 307407 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10009_solr
[junit4] 2> 307425 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307427 DEBUG (ScheduledTrigger-827-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307445 DEBUG (ScheduledTrigger-827-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307446 DEBUG (simCloudManagerPool-826-thread-4) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 307446 DEBUG (simCloudManagerPool-826-thread-4) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
4
[junit4] 2> 307446 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 307447 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 307447 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial
livenodes: [127.0.0.1:10008_solr, 127.0.0.1:10007_solr]
[junit4] 2> 307447 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 307447 DEBUG (ScheduledTrigger-827-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307447 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 307466 DEBUG (ScheduledTrigger-827-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307468 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger
with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 307468 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost
nodes: [127.0.0.1:10009_solr]
[junit4] 2> 307468 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"37dcdd09cc1caT7j5gt4o1b2my72udkkgss66hg",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":982747851833802,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982747851833802],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10009_solr"]}}
[junit4] 2> 307468 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas,
node_lost_restore_trigger]
[junit4] 2> 307468 DEBUG (ScheduledTrigger-827-thread-3) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37dcdd09cc1caT7j5gt4o1b2my72udkkgss66hg",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":982747851833802,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982747851833802],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982750897734952,
[junit4] 2> "nodeNames":["127.0.0.1:10009_solr"]}}
[junit4] 2> 307469 DEBUG (AutoscalingActionExecutor-828-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"37dcdd09cc1caT7j5gt4o1b2my72udkkgss66hg",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":982747851833802,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982747851833802],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982750897734952,
[junit4] 2> "nodeNames":["127.0.0.1:10009_solr"]}}
[junit4] 2> 307469 DEBUG (AutoscalingActionExecutor-828-thread-1) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"37dcdd09cc1caT7j5gt4o1b2my72udkkgss66hg",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":982747851833802,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[982747851833802],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":982750897734952,
[junit4] 2> "nodeNames":["127.0.0.1:10009_solr"]}}
[junit4] 2> 307469 DEBUG (AutoscalingActionExecutor-828-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_restore_trigger after
100ms
[junit4] 2> 307469 DEBUG (AutoscalingActionExecutor-828-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 307469 DEBUG (AutoscalingActionExecutor-828-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 24 ms for event
id=37dcdd09cc1caT7j5gt4o1b2my72udkkgss66hg
[junit4] 2> 307469 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exitin
[...truncated too long message...]
381d3013d855bT7j5gt4o1b2my72udkkgss64u5",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":987168189023579,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[987168189023579],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":987168209694529,
[junit4] 2> "nodeNames":["127.0.0.1:10069_solr"]}}
[junit4] 2> 19765 DEBUG (ScheduledTrigger-153-thread-3) [ ]
o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10069_solr
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"381d3013d855bT7j5gt4o1b2my72udkkgss64u5",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":987168189023579,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[987168189023579],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":987168209694529,
[junit4] 2> "nodeNames":["127.0.0.1:10069_solr"]}}
[junit4] 2> 19765 INFO (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.s.TestSimTriggerIntegration last action at 987163151309979 time =
987168238038479
[junit4] 2> 19765 INFO (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.s.TestSimTriggerIntegration action executed from node_added_trigger2
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"381d3013d855bT7j5gt4o1b2my72udkkgss64u5",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":987168189023579,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[987168189023579],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":987168209694529,
[junit4] 2> "nodeNames":["127.0.0.1:10069_solr"]}}
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms
[junit4] 2> 19765 DEBUG (AutoscalingActionExecutor-154-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 34 ms for event
id=381d3013d855bT7j5gt4o1b2my72udkkgss64u5
[junit4] 2> 19766 DEBUG (simCloudManagerPool-152-thread-5) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 19766 DEBUG (simCloudManagerPool-152-thread-5) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
5
[junit4] 2> 19767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 19767 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 19767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial
livenodes: [127.0.0.1:10069_solr, 127.0.0.1:10067_solr, 127.0.0.1:10068_solr]
[junit4] 2> 19767 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 19768 DEBUG (ScheduledTrigger-153-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 19768 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 19768 DEBUG (simCloudManagerPool-152-thread-6) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 19768 DEBUG (simCloudManagerPool-152-thread-6) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
6
[junit4] 2> 19769 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 19769 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 19769 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger2 - Initial
livenodes: [127.0.0.1:10069_solr, 127.0.0.1:10067_solr, 127.0.0.1:10068_solr]
[junit4] 2> 19769 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 19770 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 19770 DEBUG (ScheduledTrigger-153-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with
currently live nodes: 3 and last live nodes: 3
[junit4] 2> 19774 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker:
/autoscaling/nodeLost/127.0.0.1:10069_solr
[junit4] 2> 19775 INFO (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exiting.
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor
now
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread
pool executor
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 19775 DEBUG
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed
explicitly
[junit4] 2> 19776 INFO
(TEST-TestSimTriggerIntegration.testTriggerThrottling-seed#[A2EF65D9003E0910])
[ ] o.a.s.SolrTestCaseJ4 ###Ending testTriggerThrottling
[junit4] OK 0.15s J1 | TestSimTriggerIntegration.testTriggerThrottling
[junit4] 2> NOTE: leaving temporary files on disk at:
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestSimTriggerIntegration_A2EF65D9003E0910-001
[junit4] 2> NOTE: test params are: codec=Lucene70,
sim=RandomSimilarity(queryNorm=false): {}, locale=fr-RW, timezone=America/Panama
[junit4] 2> NOTE: Linux 4.18.0-15-generic amd64/Aleksey Shipilev
12-testing (64-bit)/cpus=8,threads=1,free=48847248,total=80740352
[junit4] 2> NOTE: All tests run in this JVM: [TestSimTriggerIntegration,
TestSimTriggerIntegration]
[junit4] Completed [4/5 (1!)] on J1 in 2.82s, 11 tests, 1 failure <<<
FAILURES!
[...truncated 25 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1572: The
following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1099:
There were test failures: 5 suites, 55 tests, 1 failure [seed: A2EF65D9003E0910]
Total time: 25 seconds
[repro] Setting last failure code to 256
[repro] Failures:
[repro] 1/5 failed:
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
No credentials specified
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all
warnings.
<Git Blamer> GIT_COMMIT=776013c52e58401c517b4bdd388a488520b84eb2,
workspace=/home/jenkins/workspace/Lucene-Solr-7.x-Linux
[WARNINGS] Computing warning deltas based on reference build #3589
Recording test results
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
