Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2270/
2 tests failed. FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory Error Message: expected:<5> but was:<0> Stack Trace: java.lang.AssertionError: expected:<5> but was:<0> at __randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:E20937E07A1EA8]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.failNotEquals(Assert.java:647) at org.junit.Assert.assertEquals(Assert.java:128) at org.junit.Assert.assertEquals(Assert.java:472) at org.junit.Assert.assertEquals(Assert.java:456) at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:274) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration Error Message: Path /autoscaling/nodeAdded/127.0.0.1:10010_solr should have been deleted Stack Trace: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10010_solr should have been deleted at __randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:75A425C654072C40]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertFalse(Assert.java:68) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12638 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_6D1EADCA5A32E1AF-001/init-core-data-001 [junit4] 2> 1648503 WARN (SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=41 numCloses=41 [junit4] 2> 1648504 INFO (SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 1648525 INFO (SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 1648526 INFO (SUITE-TestTriggerIntegration-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 1648527 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1648527 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1648528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 1648528 DEBUG (ScheduledTrigger-4611-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648542 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration [junit4] 2> 1648542 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1648542 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1 [junit4] 2> 1648542 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1648542 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1648543 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1648561 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1648562 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 1648565 DEBUG (ScheduledTrigger-4614-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648585 DEBUG (ScheduledTrigger-4614-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648605 DEBUG (ScheduledTrigger-4614-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648626 DEBUG (ScheduledTrigger-4614-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648646 DEBUG (ScheduledTrigger-4614-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648662 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1 [junit4] 2> 1648662 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1648663 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1648663 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10008_solr [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1648669 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 1648669 DEBUG (ScheduledTrigger-4617-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648670 DEBUG (ScheduledTrigger-4617-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr [junit4] 2> 1648690 DEBUG (ScheduledTrigger-4617-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648710 DEBUG (ScheduledTrigger-4617-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648739 DEBUG (ScheduledTrigger-4617-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648772 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS [junit4] 2> 1648776 DEBUG (ScheduledTrigger-4617-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1648793 DEBUG (simCloudManagerPool-4616-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1648793 DEBUG (simCloudManagerPool-4616-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648829 DEBUG (simCloudManagerPool-4616-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 1648829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 1648829 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 1648829 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1648841 DEBUG (simCloudManagerPool-4616-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 1648842 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1 [junit4] 2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1648842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1648842 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10010_solr at time 1144705664944789 [junit4] 2> 1648842 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 1648842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 1648845 DEBUG (ScheduledTrigger-4617-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10010_solr] added at times [1144705664944789], now=1144707034696639 [junit4] 2> 1648869 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"4111a8e880a95T38bxngjjsw1zhb3lymth2blvb", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144705664944789, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144705664944789], [junit4] 2> "nodeNames":["127.0.0.1:10010_solr"]}} [junit4] 2> 1648870 DEBUG (ScheduledTrigger-4617-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger, node_added_trigger] [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 1648950 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10008_solr [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 1648951 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeMarkersRegistration -Dtests.seed=6D1EADCA5A32E1AF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=mk -Dtests.timezone=Africa/Bamako -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] FAILURE 0.43s J2 | TestTriggerIntegration.testNodeMarkersRegistration <<< [junit4] > Throwable #1: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10010_solr should have been deleted [junit4] > at __randomizedtesting.SeedInfo.seed([6D1EADCA5A32E1AF:75A425C654072C40]:0) [junit4] > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1649017 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState [junit4] 2> 1649017 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 1649017 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10008_solr [junit4] 2> 1649017 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1649018 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1649021 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1649021 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 1649021 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649033 DEBUG (AutoscalingActionExecutor-4618-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"4111a8e880a95T38bxngjjsw1zhb3lymth2blvb", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144705664944789, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144705664944789], [junit4] 2> "_enqueue_time_":1144707060507039, [junit4] 2> "nodeNames":["127.0.0.1:10010_solr"]}} [junit4] 2> 1649033 WARN (AutoscalingActionExecutor-4618-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1649033 DEBUG (AutoscalingActionExecutor-4618-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 22 ms for event id=4111a8e880a95T38bxngjjsw1zhb3lymth2blvb [junit4] 2> 1649042 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649068 DEBUG (ScheduledTrigger-4620-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649088 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649124 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649141 DEBUG (simCloudManagerPool-4619-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1649141 DEBUG (simCloudManagerPool-4619-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 1649142 DEBUG (simCloudManagerPool-4619-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr] [junit4] 2> 1649142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 1649142 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1649142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10011_solr at time 1144721211178189 [junit4] 2> 1649153 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649173 DEBUG (ScheduledTrigger-4620-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649177 DEBUG (simCloudManagerPool-4619-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 1649178 DEBUG (simCloudManagerPool-4619-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 1649178 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1649178 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10011_solr] added at times [1144721211178189], now=1144722626030189 [junit4] 2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":1144721211178189, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144721211178189], [junit4] 2> "nodeNames":["127.0.0.1:10011_solr"]}} [junit4] 2> 1649181 DEBUG (ScheduledTrigger-4620-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger] [junit4] 2> 1649197 DEBUG (AutoscalingActionExecutor-4621-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":1144721211178189, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144721211178189], [junit4] 2> "_enqueue_time_":1144722645272739, [junit4] 2> "nodeNames":["127.0.0.1:10011_solr"]}} [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger [junit4] 2> 1649198 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 100ms [junit4] 2> 1649202 DEBUG (AutoscalingActionExecutor-4621-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 246 ms for event id=4111e2d28bccdT38bxngjjsw1zhb3lymth2blvd [junit4] 2> 1649215 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventFromRestoredState [junit4] 2> 1649215 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 1649215 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1649215 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1649216 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649226 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1 [junit4] 2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1649227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 1649227 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649247 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649303 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649327 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649336 DEBUG (simCloudManagerPool-4622-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1649336 DEBUG (simCloudManagerPool-4622-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 1649359 DEBUG (ScheduledTrigger-4623-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649359 DEBUG (simCloudManagerPool-4622-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 1649360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 1649360 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1649360 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649360 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 1144731577217889 [junit4] 2> 1649379 DEBUG (ScheduledTrigger-4623-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649380 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649380 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649400 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649426 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649426 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649444 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649446 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649446 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649464 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649466 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649466 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649497 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649497 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649497 DEBUG (ScheduledTrigger-4623-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649517 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649517 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649517 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649537 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649558 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649578 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1649598 DEBUG (ScheduledTrigger-4623-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10012_solr] added at times [1144731577217889], now=1144744501802389 [junit4] 2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"411209705f361T38bxngjjsw1zhb3lymth2blvg", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144731577217889, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144731577217889], [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 1649619 DEBUG (ScheduledTrigger-4623-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger] [junit4] 2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"411209705f361T38bxngjjsw1zhb3lymth2blvg", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144731577217889, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144731577217889], [junit4] 2> "_enqueue_time_":1144744527782839, [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms [junit4] 2> 1649620 DEBUG (AutoscalingActionExecutor-4624-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 21 ms for event id=411209705f361T38bxngjjsw1zhb3lymth2blvg [junit4] 2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649720 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10013_solr at time 1144749589232989 [junit4] 2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649740 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649768 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649769 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649769 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649789 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649809 DEBUG (ScheduledTrigger-4623-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649829 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1649829 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1649830 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10013_solr, 127.0.0.1:10012_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10013_solr, 127.0.0.1:10012_solr, 127.0.0.1:10007_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion -1 [junit4] 2> 1649890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 1649891 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1649891 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10013_solr [junit4] 2> 1649891 DEBUG (ScheduledTrigger-4626-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649906 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 1649906 DEBUG (ScheduledTrigger-4626-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649906 DEBUG (ScheduledTrigger-4626-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649931 DEBUG (ScheduledTrigger-4626-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10013_solr] added at times [1144749589232989], now=1144760139416589 [junit4] 2> 1649931 DEBUG (ScheduledTrigger-4626-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144749589232989, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144749589232989], [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"]}} [junit4] 2> 1649932 DEBUG (ScheduledTrigger-4626-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger] [junit4] 2> 1649932 DEBUG (AutoscalingActionExecutor-4627-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1144749589232989, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144749589232989], [junit4] 2> "_enqueue_time_":1144760156655139, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"]}} [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventFromRestoredState [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 4 [junit4] 2> 1649932 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 4 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 1649933 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms [junit4] 2> 1649935 DEBUG (AutoscalingActionExecutor-4627-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 162 ms for event id=41124c89f7d5dT38bxngjjsw1zhb3lymth2blvj [junit4] 2> 1649936 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue [junit4] 2> 1649936 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 1649936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 1649936 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1649936 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1649937 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1649937 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) [junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1 [junit4] 2> 1649937 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 1649938 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1649938 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 1649938 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1649967 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650004 DEBUG (ScheduledTrigger-4629-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650032 DEBUG (ScheduledTrigger-4629-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650052 DEBUG (ScheduledTrigger-4629-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650077 DEBUG (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1650077 DEBUG (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 1650088 DEBUG (ScheduledTrigger-4629-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 1650105 INFO (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 1650105 DEBUG (simCloudManagerPool-4628-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1650105 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1650106 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10014_solr at time 1144768859523039 [junit4] 2> 1650121 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10014_solr] added at times [1144768859523039], now=1144769875652839 [junit4] 2> 1650126 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"]}} [junit4] 2> 1650127 DEBUG (ScheduledTrigger-4629-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 1650141 DEBUG (AutoscalingActionExecutor-4630-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "_enqueue_time_":1144769902357589, [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"]}} [junit4] 2> 1650141 INFO (AutoscalingActionExecutor-4630-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "_enqueue_time_":1144769902357589, [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"]}} [junit4] 2> 1650142 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1650142 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1650142 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1650143 WARN (AutoscalingActionExecutor-4630-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1650144 DEBUG (AutoscalingActionExecutor-4630-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 115 ms for event id=411294538c7dfT38bxngjjsw1zhb3lymth2blvm [junit4] 2> 1650153 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10014_solr] [junit4] 2> 1650154 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10014_solr] [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1 [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 1650154 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10011_solr [junit4] 2> 1650157 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 1650157 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 1650157 DEBUG (ScheduledTrigger-4632-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650157 DEBUG (ScheduledTrigger-4632-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr [junit4] 2> 1650157 DEBUG (ScheduledTrigger-4632-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "_dequeue_time_":1144771417388089, [junit4] 2> "_enqueue_time_":1144769902357589, [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 1650157 DEBUG (ScheduledTrigger-4632-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 1650166 DEBUG (AutoscalingActionExecutor-4633-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "_dequeue_time_":1144771417388089, [junit4] 2> "_enqueue_time_":1144769902357589, [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 1650166 DEBUG (ScheduledTrigger-4632-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 1650166 INFO (AutoscalingActionExecutor-4633-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"411294538c7dfT38bxngjjsw1zhb3lymth2blvm", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1144768859523039, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1144768859523039], [junit4] 2> "_dequeue_time_":1144771417388089, [junit4] 2> "_enqueue_time_":1144769902357589, [junit4] 2> "nodeNames":["127.0.0.1:10014_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 1650166 DEBUG (ScheduledTrigger-4632-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 1650167 DEBUG (AutoscalingActionExecutor-4633-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 87 ms for event id=411294538c7dfT38bxngjjsw1zhb3lymth2blvm [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10011_solr [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 1650259 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 1650264 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate [junit4] 2> 1650264 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 1650264 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10011_solr [junit4] 2> 1650265 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 1650265 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1650265 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[6D1EADCA5A32E1AF]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10014_solr] [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion -1 [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1650281 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 1650282 DEBUG (ScheduledTrigger-4635-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650315 DEBUG (ScheduledTrigger-4635-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650372 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650393 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650425 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650445 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650448 DEBUG (simCloudManagerPool-4634-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1650448 DEBUG (simCloudManagerPool-4634-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 1650452 DEBUG (simCloudManagerPool-4634-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10014_solr] [junit4] 2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1650454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17 [junit4] 2> 1650465 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650485 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18 [junit4] 2> 1650535 DEBUG (simCloudManagerPool-4634-thread-5) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10012_solr, 127.0.0.1:10014_solr] [junit4] 2> 1650538 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18 [junit4] 2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 1650538 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18 [junit4] 2> 1650558 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650647 DEBUG (ScheduledTrigger-4635-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650738 DEBUG (ScheduledTrigger-4635-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650773 DEBUG (ScheduledTrigger-4635-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 1650775 DEBUG (ScheduledTrigger-4635-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"4112e547a9ac5T38bxngjjsw1zhb3lymth2blvq", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":1144790590331589, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10012_solr":250.0, [junit4] 2> "127.0.0.1:10014_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10012_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10014_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1m [...truncated too long message...] lrJmxReporter@2ed6b9ac: rootName = solr_33999, domain = solr.core..system.shard1.replica_n9, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n9 / com.codahale.metrics.MetricRegistry@116dc80a [junit4] 2> 3727314 INFO (coreCloseExecutor-10184-thread-2) [n:127.0.0.1:42850_solr c:AutoscalingHistoryHandlerTest_collection s:shard1 r:core_node8 x:AutoscalingHistoryHandlerTest_collection_shard1_replica_n7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.AutoscalingHistoryHandlerTest_collection.shard1.leader, tag=376278702 [junit4] 2> 3727314 INFO (jetty-closer-4750-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@719dd81b{/solr,null,UNAVAILABLE} [junit4] 2> 3727315 INFO (jetty-closer-4750-thread-2) [ ] o.e.j.s.session Stopped scavenging [junit4] 2> 3727319 INFO (jetty-closer-4750-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5c42805{/solr,null,UNAVAILABLE} [junit4] 2> 3727321 INFO (jetty-closer-4750-thread-3) [ ] o.e.j.s.session Stopped scavenging [junit4] 2> 3727324 INFO (coreCloseExecutor-10185-thread-2) [n:127.0.0.1:33999_solr c:.system s:shard1 r:core_node10 x:.system_shard1_replica_n9] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=42380608 [junit4] 2> 3727333 INFO (jetty-closer-4750-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99396511382437897-127.0.0.1:33999_solr-n_0000000000) closing [junit4] 2> 3727333 INFO (OverseerStateUpdate-99396511382437897-127.0.0.1:33999_solr-n_0000000000) [n:127.0.0.1:33999_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33999_solr [junit4] 2> 3727334 WARN (OverseerAutoScalingTriggerThread-99396511382437897-127.0.0.1:33999_solr-n_0000000000) [n:127.0.0.1:33999_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 3727335 DEBUG (jetty-closer-4750-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 3727337 INFO (jetty-closer-4750-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5c28eab1{/solr,null,UNAVAILABLE} [junit4] 2> 3727337 INFO (jetty-closer-4750-thread-1) [ ] o.e.j.s.session Stopped scavenging [junit4] 2> 3727342 ERROR (SUITE-AutoscalingHistoryHandlerTest-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 3727343 INFO (SUITE-AutoscalingHistoryHandlerTest-seed#[6D1EADCA5A32E1AF]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37461 37461 [junit4] 2> 3727423 INFO (Thread-5362) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37461 37461 [junit4] 2> 3727424 WARN (Thread-5362) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 13 /solr/aliases.json [junit4] 2> 5 /solr/configs/.system [junit4] 2> 5 /solr/configs/.system/managed-schema [junit4] 2> 4 /solr/security.json [junit4] 2> 4 /solr/configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED/managed-schema [junit4] 2> 4 /solr/configs/AutoscalingHistoryHandlerTest_collection.AUTOCREATED [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 56 /solr/collections/.system/state.json [junit4] 2> 43 /solr/collections/AutoscalingHistoryHandlerTest_collection/state.json [junit4] 2> 13 /solr/clusterprops.json [junit4] 2> 13 /solr/clusterstate.json [junit4] 2> 3 /solr/overseer_elect/election/99396511382437897-127.0.0.1:33999_solr-n_0000000000 [junit4] 2> 2 /solr/collections/.system/leader_elect/shard1/election/99396511382437895-core_node6-n_0000000000 [junit4] 2> 2 /solr/collections/AutoscalingHistoryHandlerTest_collection/leader_elect/shard1/election/99396511382437897-core_node5-n_0000000001 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 13 /solr/live_nodes [junit4] 2> 13 /solr/collections [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.handler.admin.AutoscalingHistoryHandlerTest_6D1EADCA5A32E1AF-001 [junit4] 2> Jan 23, 2018 1:20:39 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@649d4078), locale=es-US, timezone=UCT [junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=131368280,total=519569408 [junit4] 2> NOTE: All tests run in this JVM: [VMParamsZkACLAndCredentialsProvidersTest, HdfsCollectionsAPIDistributedZkTest, PropertiesRequestHandlerTest, TestCloudPseudoReturnFields, TestAnalyzeInfixSuggestions, TestQuerySenderListener, BadCopyFieldTest, SearchHandlerTest, TestLocalFSCloudBackupRestore, ClusterStateUpdateTest, TestReRankQParserPlugin, GraphQueryTest, TestFilteredDocIdSet, TestAddFieldRealTimeGet, ComputePlanActionTest, SolrShardReporterTest, TestSolrCloudWithDelegationTokens, TestPrepRecovery, TestSQLHandlerNonCloud, ResponseHeaderTest, TestSolrDeletionPolicy2, TestImplicitCoreProperties, SchemaApiFailureTest, TestSolrCoreProperties, NumericFieldsTest, TestCodecSupport, DistanceUnitsTest, TestRequestForwarding, ZkStateWriterTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestReqParamsAPI, HdfsRestartWhileUpdatingTest, SpellCheckComponentTest, TestLegacyField, TermVectorComponentTest, SpatialHeatmapFacetsTest, ClusterStateTest, LeaderFailureAfterFreshStartTest, TestEmbeddedSolrServerConstructors, MergeStrategyTest, TestUpdate, TestCloudInspectUtil, NotRequiredUniqueKeyTest, PreAnalyzedFieldTest, ExplicitHLLTest, TestExactSharedStatsCache, PeerSyncReplicationTest, SpellPossibilityIteratorTest, TestSchemaVersionResource, TestFaceting, TestSearcherReuse, DistributedFacetPivotLargeTest, TestDocBasedVersionConstraints, JvmMetricsTest, TestCollationFieldDocValues, AddBlockUpdateTest, TestSolrCloudWithHadoopAuthPlugin, TestCoreDiscovery, TestBulkSchemaConcurrent, TestTolerantSearch, MultiTermTest, TestFieldResource, TestInitQParser, TestRealTimeGet, TestAtomicUpdateErrorCases, StatsReloadRaceTest, ConnectionReuseTest, TestMissingGroups, TestSystemCollAutoCreate, TestQueryTypes, SolrGangliaReporterTest, HLLUtilTest, TestNRTOpen, TestDistributedMissingSort, BasicDistributedZk2Test, TestXmlQParser, CircularListTest, TestStressCloudBlindAtomicUpdates, TestMacroExpander, LegacyCloudClusterPropTest, ShowFileRequestHandlerTest, AliasIntegrationTest, TestPivotHelperCode, TestComputePlanAction, BasicAuthIntegrationTest, ClassificationUpdateProcessorTest, CleanupOldIndexTest, LukeRequestHandlerTest, DocValuesMultiTest, DateFieldTest, TestFastWriter, CollectionsAPISolrJTest, CoreMergeIndexesAdminHandlerTest, TestFiltering, TestShortCircuitedRequests, TestValueSourceCache, CloudMLTQParserTest, LeaderElectionTest, TestFieldCacheReopen, TestUseDocValuesAsStored2, TestTlogReplica, CurrencyRangeFacetCloudTest, HdfsNNFailoverTest, DeleteStatusTest, TestSubQueryTransformerDistrib, MultiThreadedOCPTest, TestTriggerIntegration, TestOverriddenPrefixQueryForCustomFieldType, SolrCmdDistributorTest, CollectionReloadTest, TestSolr4Spatial2, AtomicUpdatesTest, CoreAdminHandlerTest, CopyFieldTest, TestCollectionAPI, SpatialFilterTest, ConvertedLegacyTest, TestQueryWrapperFilter, UnloadDistributedZkTest, DirectoryFactoryTest, UninvertDocValuesMergePolicyTest, DistribJoinFromCollectionTest, DocValuesTest, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomDVFaceting, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, TestCoreContainer, TestSolr4Spatial, QueryElevationComponentTest, PeerSyncTest, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SoftAutoCommitTest, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, UpdateParamsTest, TestQuerySenderNoQuery, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, CursorPagingTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, ConfigSetsAPITest, CreateCollectionCleanupTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, HealthCheckHandlerTest, HttpPartitionTest, MoveReplicaTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, RecoveryAfterSoftCommitTest, TestCloudRecovery, TestLockTree, TestOnReconnectListenerSupport, TestRebalanceLeaders, ZkFailoverTest, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasIntegrationTest, SearchRateTriggerTest, TestNodeAddedTrigger, TestNodeLostTrigger, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsUnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, ConfigureRecoveryStrategyTest, QueryResultKeyTest, TestConfigSetProperties, TestCorePropertiesReload, TestReloadAndDeleteDocs, TestSolrCoreSnapshots, TestConfigReload, AutoscalingHistoryHandlerTest] [junit4] Completed [654/768 (2!)] on J2 in 39.46s, 1 test, 1 failure <<< FAILURES! [...truncated 48487 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org