Build: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-7.x-Linux/51/
Java: 64bit/jdk-10.0.1 -XX:-UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost
Error Message:
did not finish processing all events in time: started=5, finished=4
Stack Trace:
java.lang.AssertionError: did not finish processing all events in time:
started=5, finished=4
at
__randomizedtesting.SeedInfo.seed([1961A9D5677685CF:A674672BE49CE049]:0)
at org.junit.Assert.fail(Assert.java:93)
at
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.doTestNodeLost(TestLargeCluster.java:522)
at
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost(TestLargeCluster.java:375)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost
Error Message:
did not finish processing all events in time: started=5, finished=4
Stack Trace:
java.lang.AssertionError: did not finish processing all events in time:
started=5, finished=4
at
__randomizedtesting.SeedInfo.seed([1961A9D5677685CF:A674672BE49CE049]:0)
at org.junit.Assert.fail(Assert.java:93)
at
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.doTestNodeLost(TestLargeCluster.java:522)
at
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost(TestLargeCluster.java:375)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 13424 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[junit4] 2> Creating dataDir:
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_1961A9D5677685CF-001/init-core-data-001
[junit4] 2> 1025244 INFO
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true)
w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1025245 INFO
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via:
@org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0,
clientAuth=0.0/0.0)
[junit4] 2> 1025245 INFO
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks:
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1025255 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 1025255 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader:
127.0.0.1:10000_solr
[junit4] 2> 1025255 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 0
[junit4] 2> 1025255 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 1025255 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 1025256 INFO
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history
in memory.
[junit4] 2> 1025257 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr,
127.0.0.1:10017_solr, 127.0.0.1:10002_solr, 127.0.0.1:10012_solr,
127.0.0.1:10007_solr, 127.0.0.1:10015_solr, 127.0.0.1:10006_solr,
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10005_solr,
127.0.0.1:10009_solr, 127.0.0.1:10013_solr, 127.0.0.1:10000_solr,
127.0.0.1:10008_solr, 127.0.0.1:10014_solr, 127.0.0.1:10003_solr,
127.0.0.1:10011_solr]
[junit4] 2> 1025257 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025257 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 1025257 DEBUG (ScheduledTrigger-6289-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 28
[junit4] 2> 1025262 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.SolrTestCaseJ4 ###Starting testBasic
[junit4] 2> 1025262 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
1
[junit4] 2> 1025263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 1025263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1025263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 1025263 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing
object cache...
[junit4] 2> 1025263 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1025264 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1025264 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1025264 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool
executor
[junit4] 2> 1025264 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1025264 DEBUG
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1025263 WARN (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2050)
~[?:?]
[junit4] 2> at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2084)
~[?:?]
[junit4] 2> at
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189)
[java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:844) [?:?]
[junit4] 2> 1025264 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 1025264 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 2
[junit4] 2> 1025264 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 1025264 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 1025265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr,
127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr,
127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr,
127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr,
127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr,
127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr,
127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr,
127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr,
127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr,
127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr,
127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr,
127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr,
127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr,
127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr,
127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr,
127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr,
127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr,
127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr,
127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr,
127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr,
127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr,
127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr,
127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr,
127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr,
127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr,
127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr,
127.0.0.1:10072_solr]
[junit4] 2> 1025265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025265 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025265 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 1025285 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025305 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025326 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025346 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025366 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1025371 DEBUG (simCloudManagerPool-6293-thread-1) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1025371 DEBUG (simCloudManagerPool-6293-thread-1) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
3
[junit4] 2> 1025371 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 1025371 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025371 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 1025375 DEBUG (simCloudManagerPool-6293-thread-2) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1025375 DEBUG (simCloudManagerPool-6293-thread-2) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
4
[junit4] 2> 1025375 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 1025375 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1025375 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025375 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 1025403 DEBUG (simCloudManagerPool-6293-thread-4) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1:
{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":".system"}}
[junit4] 2> 1025406 DEBUG (simCloudManagerPool-6293-thread-5) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1025406 DEBUG (simCloudManagerPool-6293-thread-5) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
5
[junit4] 2> 1025407 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 1025407 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial
livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr,
127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr,
127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr,
127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr,
127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr,
127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr,
127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr,
127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr,
127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr,
127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr,
127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr,
127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr,
127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr,
127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr,
127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr,
127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr,
127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr,
127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr,
127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr,
127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr,
127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr,
127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr,
127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr,
127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr,
127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr,
127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr,
127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr,
127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr,
127.0.0.1:10072_solr]
[junit4] 2> 1025407 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025407 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 1025407 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025409 DEBUG (simCloudManagerPool-6293-thread-6) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1025409 DEBUG (simCloudManagerPool-6293-thread-6) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
6
[junit4] 2> 1025409 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 1025409 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1025409 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 1025427 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025448 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025468 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025488 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025508 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025529 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025549 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025569 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025589 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025610 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025630 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025650 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025670 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025691 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025711 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025731 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025751 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025772 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025792 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025812 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025832 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025853 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025873 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025893 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025913 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025934 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025954 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025974 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1025994 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026015 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026035 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026055 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026075 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026095 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026116 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026136 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026156 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026176 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026196 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026217 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026237 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026257 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026277 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 100
[junit4] 2> 1026280 DEBUG (simCloudManagerPool-6293-thread-85) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard4: {"core_node54":{
[junit4] 2> "core":"testBasic_shard4_replica_t54",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10055_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026280 DEBUG (simCloudManagerPool-6293-thread-84) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard3: {"core_node39":{
[junit4] 2> "core":"testBasic_shard3_replica_t39",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10049_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026280 DEBUG (simCloudManagerPool-6293-thread-83) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard2: {"core_node18":{
[junit4] 2> "core":"testBasic_shard2_replica_n18",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10052_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026280 DEBUG (simCloudManagerPool-6293-thread-86) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard5: {"core_node65":{
[junit4] 2> "core":"testBasic_shard5_replica_n65",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10087_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026280 DEBUG (simCloudManagerPool-6293-thread-87) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard1: {"core_node1":{
[junit4] 2> "core":"testBasic_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10054_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026291 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.TestLargeCluster Ready after 618ms
[junit4] 2> 1026297 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 97
[junit4] 2> 1026298 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10082_solr
[junit4] 2> 1026298 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10077_solr
[junit4] 2> 1026298 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10026_solr
[junit4] 2> 1026318 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 94
[junit4] 2> 1026318 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10016_solr
[junit4] 2> 1026318 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10083_solr
[junit4] 2> 1026318 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10058_solr
[junit4] 2> 1026327 INFO (simCloudManagerPool-6293-thread-119) [ ]
o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2581ms
[junit4] 2> 1026338 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 92
[junit4] 2> 1026338 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10049_solr
[junit4] 2> 1026338 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10073_solr
[junit4] 2> 1026358 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 92
[junit4] 2> 1026379 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 92
[junit4] 2> 1026399 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 92
[junit4] 2> 1026399 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost
nodes: [127.0.0.1:10082_solr, 127.0.0.1:10077_solr, 127.0.0.1:10026_solr]
[junit4] 2> 1026399 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"82ed6c9f3f457T9pw511u6t00f16ou7eh7prv38",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303299859772503,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303299859772503,
[junit4] 2> 2303299860425653,
[junit4] 2> 2303299860874453],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10082_solr",
[junit4] 2> "127.0.0.1:10077_solr",
[junit4] 2> "127.0.0.1:10026_solr"]}}
[junit4] 2> 1026399 DEBUG (ScheduledTrigger-6294-thread-4) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger1]
[junit4] 2> 1026402 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"82ed6c9f3f457T9pw511u6t00f16ou7eh7prv38",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303299859772503,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303299859772503,
[junit4] 2> 2303299860425653,
[junit4] 2> 2303299860874453],
[junit4] 2> "_enqueue_time_":2303304955681953,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10082_solr",
[junit4] 2> "127.0.0.1:10077_solr",
[junit4] 2> "127.0.0.1:10026_solr"]}}
[junit4] 2> 1026403 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"82ed6c9f3f457T9pw511u6t00f16ou7eh7prv38",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303299859772503,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303299859772503,
[junit4] 2> 2303299860425653,
[junit4] 2> 2303299860874453],
[junit4] 2> "_enqueue_time_":2303304955681953,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10082_solr",
[junit4] 2> "127.0.0.1:10077_solr",
[junit4] 2> "127.0.0.1:10026_solr"]}} with context properties:
{AFTER_ACTION=[start], BEFORE_ACTION=[start, compute]}
[junit4] 2> 1026450 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node74
[junit4] 2> 1026476 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node29
[junit4] 2> 1026488 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node32
[junit4] 2> 1026488 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2303309395
sessionWrapper.createTime 2303305196939953, this.sessionWrapper.createTime
2303305196939953
[junit4] 2> 1026488 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"82ed6c9f3f457T9pw511u6t00f16ou7eh7prv38",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303299859772503,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303299859772503,
[junit4] 2> 2303299860425653,
[junit4] 2> 2303299860874453],
[junit4] 2> "_enqueue_time_":2303304955681953,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10082_solr",
[junit4] 2> "127.0.0.1:10077_solr",
[junit4] 2> "127.0.0.1:10026_solr"]}} with context properties:
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2d026946,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@770b3d32,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2cce3b32],
AFTER_ACTION=[start, compute], BEFORE_ACTION=[start, compute, execute]}
[junit4] 2> 1026489 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node74
[junit4] 2> 1026589 DEBUG (simCloudManagerPool-6293-thread-124) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node77":{
[junit4] 2> "core":"testBasic_shard5_replica_p76",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10072_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 1026701 DEBUG (simCloudManagerPool-6293-thread-126) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic /
shard3: {"core_node31":{
[junit4] 2> "core":"testBasic_shard3_replica_n31",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10074_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 1026702 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node29
[junit4] 2> 1026803 DEBUG (simCloudManagerPool-6293-thread-132) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node79":{
[junit4] 2> "core":"testBasic_shard2_replica_p78",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10056_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 1026915 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node32
[junit4] 2> 1027016 DEBUG (simCloudManagerPool-6293-thread-140) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node81":{
[junit4] 2> "core":"testBasic_shard3_replica_n80",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10059_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1027129 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
[junit4] 2> 1027129 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 36348 ms for event
id=82ed6c9f3f457T9pw511u6t00f16ou7eh7prv38
[junit4] 2> 1027230 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 92
[junit4] 2> 1027230 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost
nodes: [127.0.0.1:10073_solr, 127.0.0.1:10016_solr, 127.0.0.1:10049_solr,
127.0.0.1:10058_solr, 127.0.0.1:10083_solr]
[junit4] 2> 1027230 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"82ed7436034b3T9pw511u6t00f16ou7eh7prv3b",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303301896910003,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303301896910003,
[junit4] 2> 2303300879474203,
[junit4] 2> 2303301896392503,
[junit4] 2> 2303300880739803,
[junit4] 2> 2303300880180203],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10073_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10058_solr",
[junit4] 2> "127.0.0.1:10083_solr"]}}
[junit4] 2> 1027230 DEBUG (ScheduledTrigger-6294-thread-3) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger1]
[junit4] 2> 1027232 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"82ed7436034b3T9pw511u6t00f16ou7eh7prv3b",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303301896910003,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303301896910003,
[junit4] 2> 2303300879474203,
[junit4] 2> 2303301896392503,
[junit4] 2> 2303300880739803,
[junit4] 2> 2303300880180203],
[junit4] 2> "_enqueue_time_":2303346507059053,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10073_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10058_solr",
[junit4] 2> "127.0.0.1:10083_solr"]}}
[junit4] 2> 1027232 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"82ed7436034b3T9pw511u6t00f16ou7eh7prv3b",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303301896910003,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303301896910003,
[junit4] 2> 2303300879474203,
[junit4] 2> 2303301896392503,
[junit4] 2> 2303300880739803,
[junit4] 2> 2303300880180203],
[junit4] 2> "_enqueue_time_":2303346507059053,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10073_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10058_solr",
[junit4] 2> "127.0.0.1:10083_solr"]}} with context properties:
{AFTER_ACTION=[start], BEFORE_ACTION=[start, compute]}
[junit4] 2> 1027273 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node43
[junit4] 2> 1027306 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node58
[junit4] 2> 1027323 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10094_solr&inPlaceMove=true&replica=core_node19
[junit4] 2> 1027332 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node39
[junit4] 2> 1027333 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2303351614
sessionWrapper.createTime 2303346650057453, this.sessionWrapper.createTime
2303346650057453
[junit4] 2> 1027333 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"82ed7436034b3T9pw511u6t00f16ou7eh7prv3b",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303301896910003,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303301896910003,
[junit4] 2> 2303300879474203,
[junit4] 2> 2303301896392503,
[junit4] 2> 2303300880739803,
[junit4] 2> 2303300880180203],
[junit4] 2> "_enqueue_time_":2303346507059053,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10073_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10049_solr",
[junit4] 2> "127.0.0.1:10058_solr",
[junit4] 2> "127.0.0.1:10083_solr"]}} with context properties:
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@52ece38f,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@108b7f38,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@6ec83937,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@46b5d349],
AFTER_ACTION=[start, compute], BEFORE_ACTION=[start, compute, execute]}
[junit4] 2> 1027333 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node43
[junit4] 2> 1027433 DEBUG (simCloudManagerPool-6293-thread-150) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node83":{
[junit4] 2> "core":"testBasic_shard3_replica_p82",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10053_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 1027545 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node58
[junit4] 2> 1027646 DEBUG (simCloudManagerPool-6293-thread-158) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node85":{
[junit4] 2> "core":"testBasic_shard4_replica_p84",
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10037_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 1027758 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10094_solr&inPlaceMove=true&replica=core_node19
[junit4] 2> 1027858 DEBUG (simCloudManagerPool-6293-thread-167) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node87":{
[junit4] 2> "core":"testBasic_shard2_replica_n86",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10094_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1027971 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node39
[junit4] 2> 1028071 DEBUG (simCloudManagerPool-6293-thread-179) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node89":{
[junit4] 2> "core":"testBasic_shard3_replica_t88",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10091_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 1028183 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.TestLargeCluster Ready after 92694ms
[junit4] 2> 1028184 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.TestLargeCluster OP COUNTS: {CREATE=1, DELETESTATUS=7,
MOVEREPLICA=7, REQUESTSTATUS=7, autoscaling=4, update=9}
[junit4] 2> 1028184 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
[junit4] 2> 1028184 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 47598 ms for event
id=82ed7436034b3T9pw511u6t00f16ou7eh7prv3b
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10043_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10081_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10084_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10036_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10041_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10014_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10038_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10067_solr
[junit4] 2> 1028284 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10042_solr
[junit4] 2> 1028304 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028324 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028345 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028365 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028385 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with
currently live nodes: 82
[junit4] 2> 1028385 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost
nodes: [127.0.0.1:10043_solr, 127.0.0.1:10067_solr, 127.0.0.1:10041_solr,
127.0.0.1:10042_solr, 127.0.0.1:10038_solr, 127.0.0.1:10081_solr,
127.0.0.1:10014_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr,
127.0.0.1:10011_solr]
[junit4] 2> 1028385 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"82eede9cb89f9T9pw511u6t00f16ou7eh7prv3e",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303399178242553,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303399178242553,
[junit4] 2> 2303399181321553,
[junit4] 2> 2303399179841453,
[junit4] 2> 2303399181685453,
[junit4] 2> 2303399180955453,
[junit4] 2> 2303399178730503,
[junit4] 2> 2303399180214853,
[junit4] 2> 2303399179102403,
[junit4] 2> 2303399179472153,
[junit4] 2> 2303399180586853],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10067_solr",
[junit4] 2> "127.0.0.1:10041_solr",
[junit4] 2> "127.0.0.1:10042_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10081_solr",
[junit4] 2> "127.0.0.1:10014_solr",
[junit4] 2> "127.0.0.1:10084_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}}
[junit4] 2> 1028385 DEBUG (ScheduledTrigger-6294-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger1]
[junit4] 2> 1028386 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"82eede9cb89f9T9pw511u6t00f16ou7eh7prv3e",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303399178242553,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303399178242553,
[junit4] 2> 2303399181321553,
[junit4] 2> 2303399179841453,
[junit4] 2> 2303399181685453,
[junit4] 2> 2303399180955453,
[junit4] 2> 2303399178730503,
[junit4] 2> 2303399180214853,
[junit4] 2> 2303399179102403,
[junit4] 2> 2303399179472153,
[junit4] 2> 2303399180586853],
[junit4] 2> "_enqueue_time_":2303404251537753,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10067_solr",
[junit4] 2> "127.0.0.1:10041_solr",
[junit4] 2> "127.0.0.1:10042_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10081_solr",
[junit4] 2> "127.0.0.1:10014_solr",
[junit4] 2> "127.0.0.1:10084_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}}
[junit4] 2> 1028386 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"82eede9cb89f9T9pw511u6t00f16ou7eh7prv3e",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303399178242553,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303399178242553,
[junit4] 2> 2303399181321553,
[junit4] 2> 2303399179841453,
[junit4] 2> 2303399181685453,
[junit4] 2> 2303399180955453,
[junit4] 2> 2303399178730503,
[junit4] 2> 2303399180214853,
[junit4] 2> 2303399179102403,
[junit4] 2> 2303399179472153,
[junit4] 2> 2303399180586853],
[junit4] 2> "_enqueue_time_":2303404251537753,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10067_solr",
[junit4] 2> "127.0.0.1:10041_solr",
[junit4] 2> "127.0.0.1:10042_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10081_solr",
[junit4] 2> "127.0.0.1:10014_solr",
[junit4] 2> "127.0.0.1:10084_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}} with context properties:
{AFTER_ACTION=[start], BEFORE_ACTION=[start, compute]}
[junit4] 2> 1028466 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10034_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 1028538 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10031_solr&inPlaceMove=true&replica=core_node48
[junit4] 2> 1028597 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10097_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 1028698 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node6
[junit4] 2> 1028742 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10078_solr&inPlaceMove=true&replica=core_node17
[junit4] 2> 1028779 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10075_solr&inPlaceMove=true&replica=core_node34
[junit4] 2> 1028837 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10050_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 1028863 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10024_solr&inPlaceMove=true&replica=core_node62
[junit4] 2> 1028878 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10008_solr&inPlaceMove=true&replica=core_node68
[junit4] 2> 1028886 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ComputePlanAction Computed Plan:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10062_solr&inPlaceMove=true&replica=core_node24
[junit4] 2> 1028886 INFO (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2303429307
sessionWrapper.createTime 2303404344411903, this.sessionWrapper.createTime
2303404344411903
[junit4] 2> 1028887 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"82eede9cb89f9T9pw511u6t00f16ou7eh7prv3e",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2303399178242553,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2303399178242553,
[junit4] 2> 2303399181321553,
[junit4] 2> 2303399179841453,
[junit4] 2> 2303399181685453,
[junit4] 2> 2303399180955453,
[junit4] 2> 2303399178730503,
[junit4] 2> 2303399180214853,
[junit4] 2> 2303399179102403,
[junit4] 2> 2303399179472153,
[junit4] 2> 2303399180586853],
[junit4] 2> "_enqueue_time_":2303404251537753,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10043_solr",
[junit4] 2> "127.0.0.1:10067_solr",
[junit4] 2> "127.0.0.1:10041_solr",
[junit4] 2> "127.0.0.1:10042_solr",
[junit4] 2> "127.0.0.1:10038_solr",
[junit4] 2> "127.0.0.1:10081_solr",
[junit4] 2> "127.0.0.1:10014_solr",
[junit4] 2> "127.0.0.1:10084_solr",
[junit4] 2> "127.0.0.1:10036_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}} with context properties:
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@145c86ab,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2060a749,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2285b260,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@7fa88714,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@f313f94,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2ace9bd4,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@25573a5f,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@1efc99c6,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@1837d812,
org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@4679e0fb],
AFTER_ACTION=[start, compute], BEFORE_ACTION=[start, compute, execute]}
[junit4] 2> 1028887 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10034_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 1028988 DEBUG (simCloudManagerPool-6293-thread-167) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node91":{
[junit4] 2> "core":"testBasic_shard2_replica_p90",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10034_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 1029099 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10031_solr&inPlaceMove=true&replica=core_node48
[junit4] 2> 1029200 DEBUG (simCloudManagerPool-6293-thread-195) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node93":{
[junit4] 2> "core":"testBasic_shard4_replica_n92",
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10031_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1029311 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10097_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 1029412 DEBUG (simCloudManagerPool-6293-thread-44) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node95":{
[junit4] 2> "core":"testBasic_shard1_replica_n94",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10097_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1029524 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node6
[junit4] 2> 1029624 DEBUG (simCloudManagerPool-6293-thread-14) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node97":{
[junit4] 2> "core":"testBasic_shard1_replica_t96",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 1029735 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10078_solr&inPlaceMove=true&replica=core_node17
[junit4] 2> 1029835 DEBUG (simCloudManagerPool-6293-thread-110) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node99":{
[junit4] 2> "core":"testBasic_shard2_replica_n98",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10078_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1029947 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10075_solr&inPlaceMove=true&replica=core_node34
[junit4] 2> 1030047 DEBUG (simCloudManagerPool-6293-thread-172) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node101":{
[junit4] 2> "core":"testBasic_shard3_replica_n100",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10075_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1030158 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10050_solr&inPlaceMove=true&replica=core_node4
[junit4] 2> 1030258 DEBUG (simCloudManagerPool-6293-thread-155) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node103":{
[junit4] 2> "core":"testBasic_shard1_replica_n102",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10050_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1030370 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10024_solr&inPlaceMove=true&replica=core_node62
[junit4] 2> 1030470 DEBUG (simCloudManagerPool-6293-thread-129) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node105":{
[junit4] 2> "core":"testBasic_shard5_replica_n104",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10024_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 1030581 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10008_solr&inPlaceMove=true&replica=core_node68
[junit4] 2> 1030682 DEBUG (simCloudManagerPool-6293-thread-142) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node107":{
[junit4] 2> "core":"testBasic_shard5_replica_t106",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10008_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 1030793 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10062_solr&inPlaceMove=true&replica=core_node24
[junit4] 2> 1030894 DEBUG (simCloudManagerPool-6293-thread-92) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node109":{
[junit4] 2> "core":"testBasic_shard2_replica_t108",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10062_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 1031006 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms
[junit4] 2> 1031006 DEBUG (AutoscalingActionExecutor-6295-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 130995 ms for event
id=82eede9cb89f9T9pw511u6t00f16ou7eh7prv3e
[junit4] 2> 1031023 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.TestLargeCluster Ready after 21ms
[junit4] 2> 1031023 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.TestLargeCluster ==== Flaky replicas: 38. Additional MOVEREPLICA
count: 10
[junit4] 2> 1031024 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.SolrTestCaseJ4 ###Ending testBasic
[junit4] 2> 1031026 INFO
(TEST-TestLargeCluster.testBasic-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 92
[junit4] 2> ## Empty nodes: 17
[junit4] 2> ## Dead nodes: 8
[junit4] 2> ## 127.0.0.1:10016_solr
[junit4] 2> ## 127.0.0.1:10026_solr
[junit4] 2> ## 127.0.0.1:10049_solr
[junit4] 2> ## 127.0.0.1:10058_solr
[junit4] 2> ## 127.0.0.1:10073_solr
[junit4] 2> ## 127.0.0.1:10077_solr
[junit4] 2> ## 127.0.0.1:10082_solr
[junit4] 2> ## 127.0.0.1:10083_solr
[junit4] 2> ## Collections: [.system, testBasic]
[junit4] 2> ## Max replicas per node: 2
[junit4] 2> ## Min replicas per node: 0
[junit4] 2> ## Total replicas: 76
[junit4] 2> ## * .system 1
[junit4] 2> ## - active 1
[junit4] 2> ## * testBasic 75
[junit4] 2> ## - active 75
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - DELETESTATUS 17
[junit4] 2> ## - MOVEREPLICA 17
[junit4] 2> ## - REQUESTSTATUS 17
[junit4] 2> ## - autoscaling 4
[junit4] 2> ## - update 20
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger1
[junit4] 2> ## - STARTED 3
[junit4] 2> ## - SUCCEEDED 3
[junit4] 2>
[junit4] 2> 1031074 INFO
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
7
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10011_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10082_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10014_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10049_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10036_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10016_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10038_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10077_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10041_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10083_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10042_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10026_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10043_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10073_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10067_solr
[junit4] 2> 1031074 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values:
127.0.0.1:10058_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10081_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- deleted
/autoscaling/nodeLost/127.0.0.1:10084_solr
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1031074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 1031075 INFO
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing
object cache...
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 1031075 WARN (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exiting.
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool
executor
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 1031075 DEBUG
(TEST-TestLargeCluster.testNodeLost-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 1031077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and
.scheduled_maintenance triggers
[junit4] 2> 1031077 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode
version 8
[junit4] 2> 1031078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion -1
[junit4] 2> 1031078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 1031078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial
livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10104_solr,
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr,
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr,
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr,
127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr,
127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10036_solr,
127.0.0.1:10084_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr,
127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr,
127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr,
127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr,
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10107_solr,
127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10070_solr,
127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr,
127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
127.0.0.1:10029_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr,
127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr,
127.0.0.1:10074_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr,
127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr,
127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr,
127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr,
127.0.0.1:10087_solr, 127.0.0.1:10041_solr, 127.0.0.1:10009_solr,
127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr,
127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr,
127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr,
127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr,
127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr,
127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr,
127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr,
127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10106_solr,
127.0.0.1:10100_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr,
127.0.0.1:10059_solr, 127.0.0.1:10103_solr, 127.0.0.1:10056_solr,
127.0.0.1:10072_solr]
[junit4] 2> 1031079 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1031079 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 1031079 DEBUG (ScheduledTrigger-6297-thread-2) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031100 DEBUG (ScheduledTrigger-6297-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031120 DEBUG (ScheduledTrigger-6297-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031140 DEBUG (ScheduledTrigger-6297-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031161 DEBUG (ScheduledTrigger-6297-thread-3) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031179 DEBUG (simCloudManagerPool-6296-thread-1) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1031179 DEBUG (simCloudManagerPool-6296-thread-1) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
9
[junit4] 2> 1031179 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 1031180 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1031180 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 1031181 DEBUG (ScheduledTrigger-6297-thread-4) [ ]
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with
currently live nodes: 100
[junit4] 2> 1031181 DEBUG (simCloudManagerPool-6296-thread-2) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1031181 DEBUG (simCloudManagerPool-6296-thread-2) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
10
[junit4] 2> 1031181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion
10
[junit4] 2> 1031182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 1031182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 1031182 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 1031204 DEBUG (simCloudManagerPool-6296-thread-5) [ ]
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 1031204 DEBUG (simCloudManagerPool-6296-thread-5) [ ]
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version
11
[junit4] 2> 1031204 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion
11
[junit4] 2> 1031205 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger3 - Initial
livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10104_solr,
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr,
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr,
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr,
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10105_solr,
127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10102_solr,
127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10036_solr,
127.0.0.1:10084_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr,
127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr,
127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr,
127.0.0.1:10101_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr,
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10107_solr,
127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10070_solr,
127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr,
127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr,
127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr,
127.0.0.1:10029_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr,
127.0.0.1:10064_sol
[...truncated too long message...]
\"node_name\":\"127.0.0.1:10085_solr\",\n \"state\":\"active\",\n
\"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":4,\n
\"INDEX.sizeInGB\":9.5367431640625E-6,\n \"SEARCHER.searcher.numDocs\":0,\n
\"shard\":\"shard1\",\n \"collection\":\"testSearchRate\"}}",
[junit4] 2> "{\"core_node9\":{\n
\"core\":\"testSearchRate_shard1_replica_n9\",\n
\"SEARCHER.searcher.maxDoc\":0,\n \"SEARCHER.searcher.deletedDocs\":0,\n
\"INDEX.sizeInBytes\":10240,\n \"rate\":4.0,\n
\"node_name\":\"127.0.0.1:10063_solr\",\n \"state\":\"active\",\n
\"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":4,\n
\"INDEX.sizeInGB\":9.5367431640625E-6,\n \"SEARCHER.searcher.numDocs\":0,\n
\"shard\":\"shard1\",\n \"collection\":\"testSearchRate\"}}"],
[junit4] 2> "__start__":3,
[junit4] 2> "coldNodes":{},
[junit4] 2> "coldShards":{},
[junit4] 2> "coldReplicas":[],
[junit4] 2> "coldCollections":{},
[junit4] 2> "hotShards":{"testSearchRate":{"shard1":40.0}},
[junit4] 2> "hotCollections":{"testSearchRate":40.0},
[junit4] 2> "hotNodes":{
[junit4] 2> "127.0.0.1:10001_solr":4.0,
[junit4] 2> "127.0.0.1:10085_solr":4.0,
[junit4] 2> "127.0.0.1:10090_solr":4.0,
[junit4] 2> "127.0.0.1:10089_solr":4.0,
[junit4] 2> "127.0.0.1:10086_solr":4.0,
[junit4] 2> "127.0.0.1:10063_solr":4.0,
[junit4] 2> "127.0.0.1:10092_solr":4.0,
[junit4] 2> "127.0.0.1:10093_solr":4.0,
[junit4] 2> "127.0.0.1:10060_solr":4.0,
[junit4] 2> "127.0.0.1:10023_solr":4.0},
[junit4] 2> "_enqueue_time_":2316479535593604,
[junit4] 2> "requestedOps":[
[junit4] 2> "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n
\"first\":\"testSearchRate\",\n \"second\":\"shard1\"}]}}",
[junit4] 2> "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n
\"first\":\"testSearchRate\",\n \"second\":\"shard1\"}]}}",
[junit4] 2> "Op{action=ADDREPLICA, hints={COLL_SHARD=[{\n
\"first\":\"testSearchRate\",\n \"second\":\"shard1\"}]}}"]}} with context
properties:
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@376838a7,
org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@22d1e510,
org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@436ae447],
AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 222815 DEBUG (AutoscalingActionExecutor-18-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10056_solr&type=NRT
[junit4] 2> 222836 INFO (simCloudManagerPool-16-thread-31) [ ]
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10056_solr for creating new
replica
[junit4] 2> 222870 DEBUG (AutoscalingActionExecutor-18-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10059_solr&type=NRT
[junit4] 2> 222878 INFO (simCloudManagerPool-16-thread-36) [ ]
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10059_solr for creating new
replica
[junit4] 2> 222894 DEBUG (AutoscalingActionExecutor-18-thread-1) [ ]
o.a.s.c.a.ExecutePlanAction Executing operation:
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10053_solr&type=NRT
[junit4] 2> 222894 INFO (simCloudManagerPool-16-thread-41) [ ]
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10053_solr for creating new
replica
[junit4] 2> 222908 DEBUG (simCloudManagerPool-16-thread-46) [ ]
o.a.s.c.a.s.SimClusterStateProvider -- no leader in shard1:{
[junit4] 2> "replicas":{"core_node8":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10092_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 222908 DEBUG (AutoscalingActionExecutor-18-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers Resuming trigger: search_rate_trigger after 100ms
[junit4] 2> 222908 DEBUG (AutoscalingActionExecutor-18-thread-1) [ ]
o.a.s.c.a.ScheduledTriggers -- processing took 5781 ms for event
id=83ad1fa2c1de2T9pw511u6t00f16ou7eh7prpkr
[junit4] 2> 222947 INFO
(TEST-TestLargeCluster.testSearchRate-seed#[1961A9D5677685CF]) [ ]
o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
[junit4] 2> 222948 INFO
(TEST-TestLargeCluster.testSearchRate-seed#[1961A9D5677685CF]) [ ]
o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 100
[junit4] 2> ## Empty nodes: 77
[junit4] 2> ## Dead nodes: 0
[junit4] 2> ## Collections: [.system, testSearchRate]
[junit4] 2> ## Max replicas per node: 2
[junit4] 2> ## Min replicas per node: 0
[junit4] 2> ## Total replicas: 24
[junit4] 2> ## * .system 1
[junit4] 2> ## - active 1
[junit4] 2> ## * testSearchRate 23
[junit4] 2> ## - active 23
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - ADDREPLICA 3
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - DELETESTATUS 3
[junit4] 2> ## - REQUESTSTATUS 3
[junit4] 2> ## - autoscaling 4
[junit4] 2> ## - update 2
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: search_rate_trigger
[junit4] 2> ## - STARTED 1
[junit4] 2> ## - SUCCEEDED 1
[junit4] 2>
[junit4] OK 1.74s J1 | TestLargeCluster.testSearchRate
[junit4] 2> 222949 WARN (Simulated OverseerAutoScalingTriggerThread) [
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are
closed, exiting.
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool
executor
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 222949 DEBUG
(SUITE-TestLargeCluster-seed#[1961A9D5677685CF]-worker) [ ]
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> NOTE: leaving temporary files on disk at:
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_1961A9D5677685CF-001
[junit4] 2> NOTE: test params are: codec=Lucene70,
sim=RandomSimilarity(queryNorm=true): {}, locale=dua, timezone=Asia/Qatar
[junit4] 2> NOTE: Linux 4.13.0-41-generic amd64/Oracle Corporation 10.0.1
(64-bit)/cpus=8,threads=1,free=312319040,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestLargeCluster]
[junit4] Completed [3/5 (3!)] on J1 in 222.18s, 4 tests, 1 failure <<<
FAILURES!
[...truncated 27 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/lucene/common-build.xml:1568:
The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux/lucene/common-build.xml:1092:
There were test failures: 5 suites, 20 tests, 3 failures [seed:
1961A9D5677685CF]
Total time: 4 minutes 16 seconds
[repro] Setting last failure code to 256
[repro] Failures:
[repro] 3/5 failed: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all
warnings.
<Git Blamer> GIT_COMMIT=35a7e95bce54d53820ce3e0b1e2966f609a1c1d2,
workspace=/var/lib/jenkins/workspace/Lucene-Solr-BadApples-7.x-Linux
[WARNINGS] Computing warning deltas based on reference build #50
Recording test results
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting
ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
