Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21161/ Java: 64bit/jdk-9.0.1 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed. FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState Error Message: Stack Trace: java.util.ConcurrentModificationException at __randomizedtesting.SeedInfo.seed([DFA50ED2B3DDA65:260585B6B145CFB5]:0) at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:939) at java.base/java.util.ArrayList$Itr.next(ArrayList.java:893) at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) at jdk.internal.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Build Log: [...truncated 13637 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_DFA50ED2B3DDA65-001/init-core-data-001 [junit4] 2> 2435576 INFO (SUITE-TestTriggerIntegration-seed#[DFA50ED2B3DDA65]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 2435576 INFO (SUITE-TestTriggerIntegration-seed#[DFA50ED2B3DDA65]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0) [junit4] 2> 2435577 INFO (SUITE-TestTriggerIntegration-seed#[DFA50ED2B3DDA65]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 2435577 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2435577 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 2435577 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435577 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 2435577 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 2435578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 2435578 DEBUG (ScheduledTrigger-8999-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435578 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState [junit4] 2> 2435578 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 2435578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 2435578 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2435578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435579 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2435579 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1247) [junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435579 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 2435579 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435600 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435620 DEBUG (ScheduledTrigger-9002-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435640 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435660 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435680 DEBUG (simCloudManagerPool-9001-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2435680 DEBUG (simCloudManagerPool-9001-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 2435680 DEBUG (simCloudManagerPool-9001-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435680 DEBUG (simCloudManagerPool-9001-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 2435680 DEBUG (simCloudManagerPool-9001-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435680 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 2435681 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435681 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 2435681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 2435681 DEBUG (ScheduledTrigger-9002-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 2435681 DEBUG (ScheduledTrigger-9002-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 2435701 DEBUG (ScheduledTrigger-9002-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2435701 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 2435701 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2435701 DEBUG (ScheduledTrigger-9002-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10018_solr at time 2267545810113781 [junit4] 2> 2435711 DEBUG (simCloudManagerPool-9001-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2435712 DEBUG (simCloudManagerPool-9001-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 2435712 DEBUG (simCloudManagerPool-9001-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10018_solr, 127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435712 DEBUG (simCloudManagerPool-9001-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 2435712 DEBUG (simCloudManagerPool-9001-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10018_solr, 127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435712 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 2435712 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 2435712 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10018_solr [junit4] 2> 2435712 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 2435712 DEBUG (ScheduledTrigger-9002-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 2435712 DEBUG (ScheduledTrigger-9002-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2435712 DEBUG (ScheduledTrigger-9002-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10018_solr] added at times [2267546361047081], now=2267546367930181 [junit4] 2> 2435712 DEBUG (ScheduledTrigger-9002-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e52475ca829Tbde5ashlehaufv7iykn668s28", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":2267546361047081, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267546361047081], [junit4] 2> "nodeNames":["127.0.0.1:10018_solr"]}} [junit4] 2> 2435713 DEBUG (AutoscalingActionExecutor-9003-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e52475ca829Tbde5ashlehaufv7iykn668s28", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":2267546361047081, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267546361047081], [junit4] 2> "_enqueue_time_":2267546375703231, [junit4] 2> "nodeNames":["127.0.0.1:10018_solr"]}} [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 2435713 DEBUG (AutoscalingActionExecutor-9003-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 23 ms for event id=80e52475ca829Tbde5ashlehaufv7iykn668s28 [junit4] 2> 2435713 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 2435714 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 2435716 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration [junit4] 2> 2435716 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 2435716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 2435716 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2435716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 2435716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435716 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2435716 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1247) [junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10016_solr, 127.0.0.1:10017_solr] [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 2435717 DEBUG (ScheduledTrigger-9005-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435737 DEBUG (ScheduledTrigger-9005-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435758 DEBUG (ScheduledTrigger-9005-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435778 DEBUG (ScheduledTrigger-9005-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435798 DEBUG (ScheduledTrigger-9005-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435817 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1 [junit4] 2> 2435817 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2435817 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 2435817 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10016_solr [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2435818 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 2435818 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435818 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10016_solr [junit4] 2> 2435838 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435859 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435879 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435900 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435918 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS [junit4] 2> 2435919 DEBUG (simCloudManagerPool-9007-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2435919 DEBUG (simCloudManagerPool-9007-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 2435919 DEBUG (simCloudManagerPool-9007-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435919 DEBUG (simCloudManagerPool-9007-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435919 DEBUG (simCloudManagerPool-9007-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 2435919 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 2435920 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 2435920 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2435920 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2435920 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 2435920 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10019_solr] [junit4] 2> 2435920 DEBUG (simCloudManagerPool-9007-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 2435920 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 2435920 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1 [junit4] 2> 2435921 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 2435921 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 2435921 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 2435940 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2435941 DEBUG (ScheduledTrigger-9008-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2435941 DEBUG (ScheduledTrigger-9008-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2435941 DEBUG (ScheduledTrigger-9008-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10020_solr at time 2267557788289081 [junit4] 2> 2435941 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 2435961 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2435961 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2435961 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2435961 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10020_solr] added at times [2267557788289081], now=2267558802269831 [junit4] 2> 2435961 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e54f07a8839Tbde5ashlehaufv7iykn668s2b", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2267557788289081, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267557788289081], [junit4] 2> "nodeNames":["127.0.0.1:10020_solr"]}} [junit4] 2> 2435962 DEBUG (AutoscalingActionExecutor-9009-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e54f07a8839Tbde5ashlehaufv7iykn668s2b", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2267557788289081, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267557788289081], [junit4] 2> "_enqueue_time_":2267558810116881, [junit4] 2> "nodeNames":["127.0.0.1:10020_solr"]}} [junit4] 2> 2435963 DEBUG (AutoscalingActionExecutor-9009-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 23 ms for event id=80e54f07a8839Tbde5ashlehaufv7iykn668s2b [junit4] 2> 2435981 DEBUG (ScheduledTrigger-9008-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2435982 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 2435982 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2435982 DEBUG (ScheduledTrigger-9008-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2436001 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2436002 DEBUG (ScheduledTrigger-9008-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 2436003 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2436003 DEBUG (ScheduledTrigger-9008-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2436021 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 2 [junit4] 2> 2436021 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2436021 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 2436021 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion -1 [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 2436022 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10019_solr [junit4] 2> 2436022 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 2436022 DEBUG (ScheduledTrigger-9011-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 2436022 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 2436022 DEBUG (ScheduledTrigger-9011-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr [junit4] 2> 2436022 DEBUG (ScheduledTrigger-9011-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436022 DEBUG (ScheduledTrigger-9011-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2436023 DEBUG (ScheduledTrigger-9011-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10019_solr [junit4] 2> 2436023 DEBUG (ScheduledTrigger-9011-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10019_solr] [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e55e4081a75Tbde5ashlehaufv7iykn668s2e", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2267561874430581, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267561874430581], [junit4] 2> "nodeNames":["127.0.0.1:10019_solr"]}} [junit4] 2> 2436043 DEBUG (ScheduledTrigger-9011-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 2436044 DEBUG (AutoscalingActionExecutor-9012-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e55e4081a75Tbde5ashlehaufv7iykn668s2e", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2267561874430581, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267561874430581], [junit4] 2> "_enqueue_time_":2267562904937981, [junit4] 2> "nodeNames":["127.0.0.1:10019_solr"]}} [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 2436044 DEBUG (AutoscalingActionExecutor-9012-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 13 ms for event id=80e55e4081a75Tbde5ashlehaufv7iykn668s2e [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 2 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10016_solr [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10019_solr [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 4 [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 2436044 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 2436045 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 2436045 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger [junit4] 2> 2436045 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 2436045 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 2436046 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCooldown [junit4] 2> 2436046 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 2436046 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9 [junit4] 2> 2436047 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10016_solr [junit4] 2> 2436047 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 2436047 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10019_solr [junit4] 2> 2436047 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436047 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2436047 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9 [junit4] 2> 2436047 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2057) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2436047 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2436047 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1 [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436048 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 2436048 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436068 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436089 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436109 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436129 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436148 DEBUG (simCloudManagerPool-9013-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436148 DEBUG (simCloudManagerPool-9013-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 2436148 DEBUG (simCloudManagerPool-9013-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436148 DEBUG (simCloudManagerPool-9013-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 2436148 DEBUG (simCloudManagerPool-9013-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436148 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 2436148 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 2436149 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 2436149 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436149 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 2436149 DEBUG (simCloudManagerPool-9013-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436149 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436149 DEBUG (simCloudManagerPool-9013-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 2436149 DEBUG (simCloudManagerPool-9013-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436150 DEBUG (simCloudManagerPool-9013-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 2436150 DEBUG (simCloudManagerPool-9013-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10020_solr] [junit4] 2> 2436150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 2436150 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 2436169 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436169 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2436169 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10021_solr at time 2267569212618431 [junit4] 2> 2436169 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2436189 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436190 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2436190 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10021_solr] added at times [2267569212618431], now=2267570231045781 [junit4] 2> 2436190 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2436190 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e57996bf6bfTbde5ashlehaufv7iykn668s2h", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267569212618431, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267569212618431], [junit4] 2> "nodeNames":["127.0.0.1:10021_solr"]}} [junit4] 2> 2436190 DEBUG (AutoscalingActionExecutor-9015-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e57996bf6bfTbde5ashlehaufv7iykn668s2h", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267569212618431, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267569212618431], [junit4] 2> "_enqueue_time_":2267570240038431, [junit4] 2> "nodeNames":["127.0.0.1:10021_solr"]}} [junit4] 2> 2436191 DEBUG (AutoscalingActionExecutor-9015-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 53 ms for event id=80e57996bf6bfTbde5ashlehaufv7iykn668s2h [junit4] 2> 2436210 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 2436210 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436210 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 2436230 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436230 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436230 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436231 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10022_solr at time 2267572278323731 [junit4] 2> 2436250 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436251 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436251 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436251 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10022_solr] added at times [2267572278323731], now=2267573291345181 [junit4] 2> 2436251 DEBUG (ScheduledTrigger-9014-thread-2) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: { [junit4] 2> "id":"80e585026ea13Tbde5ashlehaufv7iykn668s2k", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267572278323731, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267572278323731], [junit4] 2> "nodeNames":["127.0.0.1:10022_solr"]}} [junit4] 2> 2436270 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436272 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436272 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436272 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10022_solr] added at times [2267572278323731], now=2267574336184631 [junit4] 2> 2436272 DEBUG (ScheduledTrigger-9014-thread-4) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: { [junit4] 2> "id":"80e585026ea13Tbde5ashlehaufv7iykn668s2m", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267572278323731, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267572278323731], [junit4] 2> "nodeNames":["127.0.0.1:10022_solr"]}} [junit4] 2> 2436291 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436294 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436294 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436294 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10022_solr] added at times [2267572278323731], now=2267575469076631 [junit4] 2> 2436294 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e585026ea13Tbde5ashlehaufv7iykn668s2o", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267572278323731, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267572278323731], [junit4] 2> "nodeNames":["127.0.0.1:10022_solr"]}} [junit4] 2> 2436299 DEBUG (AutoscalingActionExecutor-9015-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e585026ea13Tbde5ashlehaufv7iykn668s2o", [junit4] 2> "source":"node_added_cooldown_trigger", [junit4] 2> "eventTime":2267572278323731, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267572278323731], [junit4] 2> "_enqueue_time_":2267575476802981, [junit4] 2> "nodeNames":["127.0.0.1:10022_solr"]}} [junit4] 2> 2436300 DEBUG (AutoscalingActionExecutor-9015-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 47 ms for event id=80e585026ea13Tbde5ashlehaufv7iykn668s2o [junit4] 2> 2436311 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436319 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436319 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436332 DEBUG (ScheduledTrigger-9014-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 2436339 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger [junit4] 2> 2436339 DEBUG (ScheduledTrigger-9014-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCooldown [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 4 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 4 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 6 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_cooldown_trigger [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - IGNORED 2 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 2436340 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 2 [junit4] 2> 2436342 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger [junit4] 2> 2436342 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 2436342 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 2436342 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 2436342 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2436342 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436342 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1247) [junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2436343 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10021_solr, 127.0.0.1:10017_solr] [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1 [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436345 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 2436345 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436367 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436388 DEBUG (ScheduledTrigger-9017-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436408 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436428 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436445 DEBUG (simCloudManagerPool-9016-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436445 DEBUG (simCloudManagerPool-9016-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 2436446 DEBUG (simCloudManagerPool-9016-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10021_solr, 127.0.0.1:10017_solr] [junit4] 2> 2436446 DEBUG (simCloudManagerPool-9016-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10021_solr, 127.0.0.1:10017_solr] [junit4] 2> 2436446 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 2436446 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 2436446 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436446 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15 [junit4] 2> 2436446 DEBUG (ScheduledTrigger-9017-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 2436446 DEBUG (ScheduledTrigger-9017-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr [junit4] 2> 2436448 DEBUG (ScheduledTrigger-9017-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 2436448 DEBUG (ScheduledTrigger-9017-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr [junit4] 2> 2436466 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 2436467 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10021_solr] [junit4] 2> 2436467 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e5ad291c061Tbde5ashlehaufv7iykn668s2r", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2267583056298081, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267583056298081], [junit4] 2> "nodeNames":["127.0.0.1:10021_solr"]}} [junit4] 2> 2436467 DEBUG (AutoscalingActionExecutor-9018-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e5ad291c061Tbde5ashlehaufv7iykn668s2r", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2267583056298081, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2267583056298081], [junit4] 2> "_enqueue_time_":2267584084123981, [junit4] 2> "nodeNames":["127.0.0.1:10021_solr"]}} [junit4] 2> 2436467 DEBUG (AutoscalingActionExecutor-9018-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 22 ms for event id=80e5ad291c061Tbde5ashlehaufv7iykn668s2r [junit4] 2> 2436468 DEBUG (simCloudManagerPool-9016-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436468 DEBUG (simCloudManagerPool-9016-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 2436468 DEBUG (simCloudManagerPool-9016-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr] [junit4] 2> 2436468 DEBUG (simCloudManagerPool-9016-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr] [junit4] 2> 2436468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 2436468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 2436468 DEBUG (ScheduledTrigger-9017-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 2436487 DEBUG (ScheduledTrigger-9017-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 2436488 DEBUG (ScheduledTrigger-9017-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 2436507 DEBUG (ScheduledTrigger-9017-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 1 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 1 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10021_solr [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 2436508 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 2436509 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 2436509 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 2436509 DEBUG (ScheduledTrigger-9017-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 2436509 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger [junit4] 2> 2436509 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 2436509 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 2436510 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate [junit4] 2> 2436510 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 2436510 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 2436510 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10021_solr [junit4] 2> 2436510 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 2436510 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436511 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 2436511 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1247) [junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18 [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10023_solr] [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion -1 [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18 [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18 [junit4] 2> 2436511 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436532 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436552 DEBUG (ScheduledTrigger-9020-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436572 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436592 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436612 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436633 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436643 DEBUG (simCloudManagerPool-9019-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436643 DEBUG (simCloudManagerPool-9019-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 19 [junit4] 2> 2436644 DEBUG (simCloudManagerPool-9019-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10023_solr] [junit4] 2> 2436644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19 [junit4] 2> 2436644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19 [junit4] 2> 2436645 DEBUG (simCloudManagerPool-9019-thread-5) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 2436645 DEBUG (simCloudManagerPool-9019-thread-5) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20 [junit4] 2> 2436645 DEBUG (simCloudManagerPool-9019-thread-5) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10017_solr, 127.0.0.1:10023_solr] [junit4] 2> 2436645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20 [junit4] 2> 2436645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 2436645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20 [junit4] 2> 2436653 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436673 DEBUG (ScheduledTrigger-9020-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436687 DEBUG (ScheduledTrigger-9020-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"80e5d6030cad7Tbde5ashlehaufv7iykn668s2u", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":2267594022243031, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10017_solr":250.0, [junit4] 2> "127.0.0.1:10023_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10017_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10023_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}}} [junit4] 2> 2436688 DEBUG (AutoscalingActionExecutor-9021-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"80e5d6030cad7Tbde5ashlehaufv7iykn668s2u", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":2267594022243031, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10017_solr":250.0, [junit4] 2> "127.0.0.1:10023_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10017_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10023_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}, [junit4] 2> "_enqueue_time_":2267595114325181}} [junit4] 2> 2436689 DEBUG (AutoscalingActionExecutor-9021-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 53 ms for event id=80e5d6030cad7Tbde5ashlehaufv7iykn668s2u [junit4] 2> 2436693 DEBUG (ScheduledTrigger-9020-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436708 DEBUG (ScheduledTrigger-9020-thread-4) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: { [junit4] 2> "id":"80e5da0020d83Tbde5ashlehaufv7iykn668s2x", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":2267595092921731, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10017_solr":250.0, [junit4] 2> "127.0.0.1:10023_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10017_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10023_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}}} [junit4] 2> 2436713 DEBUG (ScheduledTrigger-9020-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 2436729 DEBUG (ScheduledTrigger-9020-thread-2) [ ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: { [junit4] 2> "id":"80e5da0020d83Tbde5ashlehaufv7iykn668s2z", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":2267595092921731, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10017_solr":250.0, [junit4] 2> "127.0.0.1:10023_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10017_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10023_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}}} [junit4] 2> 2436729 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSearchRate [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 2436730 INFO (TEST-TestTriggerIntegr [...truncated too long message...] ion.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 2438777 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 2438777 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10031_solr [junit4] 2> 2438777 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 2438777 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 2438778 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[DFA50ED2B3DDA65]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 2438778 DEBUG (AutoscalingActionExecutor-9045-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 75 ms for event id=80e759247ea5bTbde5ashlehaufv7iykn668s4e [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeLostTriggerRestoreState -Dtests.seed=DFA50ED2B3DDA65 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=lt -Dtests.timezone=Antarctica/McMurdo -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] ERROR 0.15s J0 | TestTriggerIntegration.testNodeLostTriggerRestoreState <<< [junit4] > Throwable #1: java.util.ConcurrentModificationException [junit4] > at __randomizedtesting.SeedInfo.seed([DFA50ED2B3DDA65:260585B6B145CFB5]:0) [junit4] > at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:939) [junit4] > at java.base/java.util.ArrayList$Itr.next(ArrayList.java:893) [junit4] > at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) [junit4] > at jdk.internal.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) [junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564) [junit4] > at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 2438780 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 2438781 DEBUG (SUITE-TestTriggerIntegration-seed#[DFA50ED2B3DDA65]-worker) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_DFA50ED2B3DDA65-001 [junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4952b43f), locale=lt, timezone=Antarctica/McMurdo [junit4] 2> NOTE: Linux 4.10.0-40-generic amd64/Oracle Corporation 9.0.1 (64-bit)/cpus=8,threads=1,free=283891824,total=536870912 [junit4] 2> NOTE: All tests run in this JVM: [TestNamedUpdateProcessors, UpdateRequestProcessorFactoryTest, TestSolrConfigHandlerConcurrent, MetricsHandlerTest, CoreAdminRequestStatusTest, TestSolrFieldCacheBean, ZkStateReaderTest, TestPostingsSolrHighlighter, OverseerCollectionConfigSetProcessorTest, SmileWriterTest, HdfsRecoveryZkTest, MetricUtilsTest, DeleteReplicaTest, FileUtilsTest, PrimitiveFieldTypeTest, HdfsDirectoryFactoryTest, NoCacheHeaderTest, SharedFSAutoReplicaFailoverTest, TestSegmentSorting, TestMaxScoreQueryParser, SSLMigrationTest, TestClassicSimilarityFactory, AnalyticsMergeStrategyTest, TestChildDocTransformer, TestLegacyFieldCache, TermVectorComponentTest, TestMiniSolrCloudClusterSSL, DateFieldTest, ComputePlanActionTest, SpellCheckCollatorWithCollapseTest, SolrShardReporterTest, TestImpersonationWithHadoopAuth, HdfsBasicDistributedZkTest, HdfsWriteToMultipleCollectionsTest, TestComplexPhraseLeadingWildcard, TestSolrXml, SyncSliceTest, TestCollectionsAPIViaSolrCloudCluster, TestMacroExpander, TestReloadDeadlock, URLClassifyProcessorTest, SolrIndexMetricsTest, CursorPagingTest, ZkControllerTest, TestFieldCacheSortRandom, BlockCacheTest, CdcrUpdateLogTest, DistributedFacetPivotSmallTest, TestLRUStatsCache, TestNodeLostTrigger, TestUpdate, TestComponentsName, HttpSolrCallGetCoreTest, SearchHandlerTest, SliceStateTest, TestComputePlanAction, UninvertDocValuesMergePolicyTest, OverseerStatusTest, TestSchemalessBufferedUpdates, ResourceLoaderTest, ReplicationFactorTest, TestPayloadScoreQParserPlugin, TestGenericDistributedQueue, CollectionStateFormat2Test, CdcrBootstrapTest, LargeFieldTest, MultiTermTest, TestSQLHandler, TestSmileRequest, SolrJmxReporterTest, TestConfigSets, AnalysisAfterCoreReloadTest, TestEmbeddedSolrServerSchemaAPI, SolrCoreTest, TestTrieFacet, CollectionsAPISolrJTest, StatelessScriptUpdateProcessorFactoryTest, TestHalfAndHalfDocValues, TestLeaderInitiatedRecoveryThread, BlockJoinFacetSimpleTest, TestReplicationHandler, DateMathParserTest, TestMinMaxOnMultiValuedField, SpellCheckCollatorTest, UUIDUpdateProcessorFallbackTest, SolrInfoBeanTest, TestRandomRequestDistribution, TestHdfsUpdateLog, HdfsCollectionsAPIDistributedZkTest, TestDistributedSearch, TestManagedSchemaThreadSafety, TestConfigReload, ExternalFileFieldSortTest, TestSort, TestDistributedStatsComponentCardinality, LeaderInitiatedRecoveryOnCommitTest, RequestLoggingTest, TestLegacyField, TestMaxTokenLenTokenizer, TestHdfsCloudBackupRestore, MoveReplicaHDFSTest, TestSolrCloudWithSecureImpersonation, TestCSVLoader, UpdateLogTest, TestSubQueryTransformerDistrib, TestFastOutputStream, TestShardHandlerFactory, DistributedMLTComponentTest, TestReversedWildcardFilterFactory, HLLSerializationTest, DistribCursorPagingTest, TestCollectionAPIs, TestHashQParserPlugin, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestDistribIDF, TestReplicationHandlerBackup, TestFieldSortValues, DeleteInactiveReplicaTest, TestNestedDocsSort, TestDFRSimilarityFactory, TestUninvertingReader, DirectUpdateHandlerOptimizeTest, TestSolrIndexConfig, DistributedQueryElevationComponentTest, BasicDistributedZkTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, TestRandomFaceting, TestFaceting, TestRecovery, TestStressVersions, SpellCheckComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, TestFiltering, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, TestFoldingMultitermQuery, DocValuesTest, DocValuesMultiTest, CacheHeaderTest, TestQueryUtils, DocumentAnalysisRequestHandlerTest, TestQueryTypes, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, TestIndexingPerformance, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCollationField, MBeansHandlerTest, SOLR749Test, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestBinaryField, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestSolrCoreProperties, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TimeZoneUtilsTest, ScriptEngineTest, PluginInfoTest, TestFastLRUCache, PrimUtilsTest, TestSuggestSpellingConverter, DOMUtilTest, RAMDirectoryFactoryTest, TestSolrJ, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SystemInfoHandlerTest, CircularListTest, TestRTGBase, DistributedIntervalFacetingTest, ActionThrottleTest, AliasIntegrationTest, AssignTest, CollectionReloadTest, ConnectionManagerTest, DeleteStatusTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, ForceLeaderTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudSearcherWarming, TestDeleteCollectionOnDownNodes, TestLeaderElectionZkExpiry, TestOnReconnectListenerSupport, TestRandomFlRTGCloud, TestRequestStatusCollectionAPI, TestUtilizeNode, TriLevelCompositeIdRoutingTest, HdfsAutoAddReplicasIntegrationTest, SearchRateTriggerTest, TestExecutePlanAction, TestTriggerIntegration] [junit4] Completed [602/765 (1!)] on J0 in 3.22s, 11 tests, 1 error <<< FAILURES! [...truncated 41396 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org