Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/375/ Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation Error Message: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=37645, name=jetty-launcher-8147-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=37645, name=jetty-launcher-8147-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) at __randomizedtesting.SeedInfo.seed([A5B9F22229685486]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) at __randomizedtesting.SeedInfo.seed([A5B9F22229685486]:0) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState Error Message: The trigger did not fire at all Stack Trace: java.lang.AssertionError: The trigger did not fire at all at __randomizedtesting.SeedInfo.seed([A5B9F22229685486:8E462779B3104156]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState(TestTriggerIntegration.java:307) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12063 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_A5B9F22229685486-001/init-core-data-001 [junit4] 2> 714550 WARN (SUITE-TestTriggerIntegration-seed#[A5B9F22229685486]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3 [junit4] 2> 714553 INFO (SUITE-TestTriggerIntegration-seed#[A5B9F22229685486]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 714555 INFO (SUITE-TestTriggerIntegration-seed#[A5B9F22229685486]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) [junit4] 2> 714555 INFO (SUITE-TestTriggerIntegration-seed#[A5B9F22229685486]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 714645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 714645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 714645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 714645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 714645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 714653 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 714653 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 714657 DEBUG (ScheduledTrigger-1947-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714714 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration [junit4] 2> 714718 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 714718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 714722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 714722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 714722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1 [junit4] 2> 714732 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 714733 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 714734 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 714734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 714734 DEBUG (ScheduledTrigger-1950-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714754 DEBUG (ScheduledTrigger-1950-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714775 DEBUG (ScheduledTrigger-1950-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714797 DEBUG (ScheduledTrigger-1950-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714819 DEBUG (ScheduledTrigger-1950-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714835 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1 [junit4] 2> 714835 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 714836 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 714837 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 714837 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10001_solr [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 714838 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 714838 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714838 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr [junit4] 2> 714858 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714879 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714900 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714920 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714948 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714948 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS [junit4] 2> 714968 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714968 DEBUG (simCloudManagerPool-1952-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 714969 DEBUG (simCloudManagerPool-1952-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 714969 DEBUG (simCloudManagerPool-1952-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714969 DEBUG (simCloudManagerPool-1952-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714969 DEBUG (simCloudManagerPool-1952-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 714969 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 714970 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 714986 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 714988 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 714988 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 714989 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 714987 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 714989 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 714989 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714989 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714989 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 714989 DEBUG (simCloudManagerPool-1952-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 714989 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1 [junit4] 2> 714990 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 714990 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 714990 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 714990 DEBUG (ScheduledTrigger-1953-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 715009 DEBUG (ScheduledTrigger-1953-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715011 DEBUG (ScheduledTrigger-1953-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 715011 DEBUG (ScheduledTrigger-1953-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715011 DEBUG (ScheduledTrigger-1953-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715011 DEBUG (ScheduledTrigger-1953-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 2904833402273924 [junit4] 2> 715032 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715033 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 715033 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715033 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715033 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [2904833402273924], now=2904834511428874 [junit4] 2> 715033 DEBUG (ScheduledTrigger-1953-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51ee3b130084Tahighjtruulsnbyv23v5sfeb0", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904833402273924, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904833402273924], [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 715036 DEBUG (AutoscalingActionExecutor-1954-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a51ee3b130084Tahighjtruulsnbyv23v5sfeb0", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904833402273924, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904833402273924], [junit4] 2> "_enqueue_time_":2904834567956524, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 715039 DEBUG (AutoscalingActionExecutor-1954-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 112 ms for event id=a51ee3b130084Tahighjtruulsnbyv23v5sfeb0 [junit4] 2> 715059 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715059 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 715059 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715059 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715079 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715079 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 715079 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715079 DEBUG (ScheduledTrigger-1953-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715090 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 2 [junit4] 2> 715090 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 715091 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 715092 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion -1 [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 715093 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 715093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10000_solr [junit4] 2> 715094 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 715094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715094 DEBUG (ScheduledTrigger-1956-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr [junit4] 2> 715115 DEBUG (ScheduledTrigger-1956-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 715121 DEBUG (ScheduledTrigger-1956-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10000_solr] [junit4] 2> 715121 DEBUG (ScheduledTrigger-1956-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51ef32def11aTahighjtruulsnbyv23v5sfeb3", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2904837559611674, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904837559611674], [junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} [junit4] 2> 715122 DEBUG (AutoscalingActionExecutor-1957-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a51ef32def11aTahighjtruulsnbyv23v5sfeb3", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2904837559611674, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904837559611674], [junit4] 2> "_enqueue_time_":2904838916709424, [junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 715122 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 2 [junit4] 2> 715123 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10000_solr [junit4] 2> 715123 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10001_solr [junit4] 2> 715124 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 715124 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 715124 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 715124 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 715125 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 715125 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 715125 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 715125 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 715127 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 715127 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 715128 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 715128 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger [junit4] 2> 715128 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 715129 DEBUG (ScheduledTrigger-1956-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715129 DEBUG (ScheduledTrigger-1956-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 715133 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState [junit4] 2> 715133 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 715133 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 715133 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr [junit4] 2> 715133 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr [junit4] 2> 715133 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 715135 DEBUG (AutoscalingActionExecutor-1957-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 647 ms for event id=a51ef32def11aTahighjtruulsnbyv23v5sfeb3 [junit4] 2> 715135 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 715135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 715135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 715136 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) [junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 715136 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715157 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715179 DEBUG (ScheduledTrigger-1959-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715199 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715231 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715244 DEBUG (simCloudManagerPool-1958-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 715244 DEBUG (simCloudManagerPool-1958-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 715245 DEBUG (simCloudManagerPool-1958-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715245 DEBUG (simCloudManagerPool-1958-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 715245 DEBUG (simCloudManagerPool-1958-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 715245 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 715246 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 715246 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 715246 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715246 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 2904845154404774 [junit4] 2> 715251 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715266 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 715270 DEBUG (ScheduledTrigger-1959-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715272 DEBUG (ScheduledTrigger-1959-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715277 DEBUG (simCloudManagerPool-1958-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 715278 DEBUG (simCloudManagerPool-1958-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 715278 DEBUG (simCloudManagerPool-1958-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715278 DEBUG (simCloudManagerPool-1958-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 715278 DEBUG (simCloudManagerPool-1958-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 715278 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 715279 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 715279 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 715280 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 715280 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715280 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [2904845154404774], now=2904846854584724 [junit4] 2> 715280 DEBUG (ScheduledTrigger-1959-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51f0f78e49a6Tahighjtruulsnbyv23v5sfeb6", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":2904845154404774, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904845154404774], [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 715281 DEBUG (AutoscalingActionExecutor-1960-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a51f0f78e49a6Tahighjtruulsnbyv23v5sfeb6", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":2904845154404774, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904845154404774], [junit4] 2> "_enqueue_time_":2904846862069324, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 715281 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 715282 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 715283 DEBUG (AutoscalingActionExecutor-1960-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 98 ms for event id=a51f0f78e49a6Tahighjtruulsnbyv23v5sfeb6 [junit4] 2> 715287 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState [junit4] 2> 715287 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 715287 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 715288 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 715303 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1 [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715304 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 715304 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715325 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715351 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715375 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715409 DEBUG (simCloudManagerPool-1961-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 715409 DEBUG (simCloudManagerPool-1961-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 715430 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715433 DEBUG (simCloudManagerPool-1961-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr] [junit4] 2> 715433 DEBUG (simCloudManagerPool-1961-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr] [junit4] 2> 715433 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 715433 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 715433 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715433 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 715434 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 [junit4] 2> 715434 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr [junit4] 2> 715451 DEBUG (ScheduledTrigger-1962-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715451 DEBUG (ScheduledTrigger-1962-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr [junit4] 2> 715506 DEBUG (ScheduledTrigger-1962-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715525 DEBUG (simCloudManagerPool-1961-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 715551 DEBUG (simCloudManagerPool-1961-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 715551 DEBUG (simCloudManagerPool-1961-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715551 DEBUG (simCloudManagerPool-1961-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 715552 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 [junit4] 2> 715552 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10005_solr] [junit4] 2> 715552 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 715552 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10005_solr [junit4] 2> 715552 DEBUG (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51f32918ae42Tahighjtruulsnbyv23v5sfeb9", [junit4] 2> "source":"node_lost_restore_trigger", [junit4] 2> "eventTime":2904854575492674, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904854575492674], [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 715552 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715552 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 715552 ERROR (ScheduledTrigger-1962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Unexpected exception in NodeLostTrigger [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_lost_restore_trigger has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:487) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$4(ScheduledTriggers.java:268) [junit4] 2> at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:159) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:537) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 715553 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 [junit4] 2> 715553 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10005_solr] [junit4] 2> 715553 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51f4888afcb2Tahighjtruulsnbyv23v5sfeba", [junit4] 2> "source":"node_lost_restore_trigger", [junit4] 2> "eventTime":2904860471786674, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904860471786674], [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 715554 DEBUG (ScheduledTrigger-1962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:10005_solr] is not ready, will try later [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10005_solr [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_restore_trigger [junit4] 2> 715653 INFO (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - IGNORED 1 [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeLostTriggerRestoreState -Dtests.seed=A5B9F22229685486 -Dtests.slow=true -Dtests.locale=no -Dtests.timezone=AET -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] FAILURE 0.39s J0 | TestTriggerIntegration.testNodeLostTriggerRestoreState <<< [junit4] > Throwable #1: java.lang.AssertionError: The trigger did not fire at all [junit4] > at __randomizedtesting.SeedInfo.seed([A5B9F22229685486:8E462779B3104156]:0) [junit4] > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeLostTriggerRestoreState(TestTriggerIntegration.java:307) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] 2> 715680 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventFromRestoredState [junit4] 2> 715681 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 715681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 715681 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10005_solr [junit4] 2> 715681 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 715681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 715681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715681 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) [junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 715682 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 715682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 715682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 715682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1 [junit4] 2> 715682 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 715683 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 715683 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 715683 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715703 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715723 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715744 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715771 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 715787 DEBUG (simCloudManagerPool-1964-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 715788 DEBUG (simCloudManagerPool-1964-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 715788 DEBUG (simCloudManagerPool-1964-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715788 DEBUG (simCloudManagerPool-1964-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 715788 DEBUG (simCloudManagerPool-1964-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 715788 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 715789 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 715789 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15 [junit4] 2> 715789 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715789 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715789 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10006_solr at time 2904872331161124 [junit4] 2> 715793 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715820 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715820 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715820 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715843 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715843 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715843 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715865 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715865 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715865 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715893 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715893 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715893 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715913 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715913 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715913 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715933 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715933 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715933 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715955 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715955 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715955 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715975 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 715979 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 715979 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 715996 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 716007 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716007 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 716011 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10006_solr] added at times [2904872331161124], now=2904883425110974 [junit4] 2> 716011 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51f74b6aae24Tahighjtruulsnbyv23v5sfebc", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904872331161124, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904872331161124], [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 716012 DEBUG (AutoscalingActionExecutor-1966-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a51f74b6aae24Tahighjtruulsnbyv23v5sfebc", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904872331161124, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904872331161124], [junit4] 2> "_enqueue_time_":2904883437274624, [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 716014 DEBUG (AutoscalingActionExecutor-1966-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 80 ms for event id=a51f74b6aae24Tahighjtruulsnbyv23v5sfebc [junit4] 2> 716016 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716033 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716033 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716034 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 2904884551424924 [junit4] 2> 716037 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716054 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716054 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716058 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716074 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716074 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716079 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716095 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716095 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716099 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716116 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716116 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716122 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716137 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716137 DEBUG (ScheduledTrigger-1965-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716143 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716165 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716165 DEBUG (ScheduledTrigger-1965-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716165 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716186 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716186 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716186 DEBUG (ScheduledTrigger-1965-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716211 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716211 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716211 DEBUG (ScheduledTrigger-1965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716213 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 716213 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 716214 DEBUG (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 716223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 716224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10006_solr, 127.0.0.1:10002_solr, 127.0.0.1:10007_solr] [junit4] 2> 716224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10006_solr, 127.0.0.1:10002_solr, 127.0.0.1:10007_solr] [junit4] 2> 716224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 716224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion -1 [junit4] 2> 716224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 716227 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 716227 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10007_solr [junit4] 2> 716227 DEBUG (ScheduledTrigger-1968-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 716235 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15 [junit4] 2> 716235 DEBUG (ScheduledTrigger-1968-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 716235 DEBUG (ScheduledTrigger-1968-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4 [junit4] 2> 716235 DEBUG (ScheduledTrigger-1968-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10007_solr] added at times [2904884551424924], now=2904894638478574 [junit4] 2> 716235 DEBUG (ScheduledTrigger-1968-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a51fa23cd1b9cTahighjtruulsnbyv23v5sfebf", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904884551424924, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904884551424924], [junit4] 2> "nodeNames":["127.0.0.1:10007_solr"]}} [junit4] 2> 716236 DEBUG (AutoscalingActionExecutor-1969-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a51fa23cd1b9cTahighjtruulsnbyv23v5sfebf", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2904884551424924, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2904884551424924], [junit4] 2> "_enqueue_time_":2904894646458224, [junit4] 2> "nodeNames":["127.0.0.1:10007_solr"]}} [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventFromRestoredState [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 716236 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 4 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 4 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 716237 INFO (TEST-TestTriggerIntegration.testEventFromRestoredState-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 716239 DEBUG (AutoscalingActionExecutor-1969-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 147 ms for event id=a51fa23cd1b9cTahighjtruulsnbyv23v5sfebf [junit4] 2> 716242 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[A5B9F22229685486]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCooldown [junit4] 2> 716242 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 716242 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 716244 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[A5B9F22229685486]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 716245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 716245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 716245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10002_solr] [junit4] 2> 716245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion -1 [junit4] 2> 716245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 716246 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 716246 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17 [junit4] 2> 716246 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716248 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 716266 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716286 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716306 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716326 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716345 DEBUG (simCloudManagerPool-1970-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 716346 DEBUG (simCloudManagerPool-1970-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18 [junit4] 2> 716352 DEBUG (ScheduledTrigger-1971-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 716352 DEBUG (simCloudManagerPool-1970-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10002_solr] [junit4] 2> 716352 DEBUG (simCloudManagerPool-1970-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 716352 DEBUG (simCloudManagerPool-1970-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10002_solr] [junit4] 2> 716352 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18 [junit4] 2> 716352 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 716352 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18 [junit4] 2> 716352 DEBUG (ScheduledTrigger-1971-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Runn [...truncated too long message...] onentTest, CloneFieldUpdateProcessorFactoryTest, TestDistributedMissingSort, TestSolrCloudSnapshots, TestAnalyzeInfixSuggestions, LegacyCloudClusterPropTest, TestZkChroot, TestSimpleTrackingShardHandler, SimpleCollectionCreateDeleteTest, TestTolerantUpdateProcessorCloud, HttpSolrCallGetCoreTest, ManagedSchemaRoundRobinCloudTest, TestFilteredDocIdSet, ZkFailoverTest, PropertiesRequestHandlerTest, TestFunctionQuery, TestSchemalessBufferedUpdates, SpellPossibilityIteratorTest, TestMacroExpander, SimpleFacetsTest, PolyFieldTest, TestSuggestSpellingConverter, TestJsonFacetRefinement, ExitableDirectoryReaderTest, TestFieldCollectionResource, EchoParamsTest, MissingSegmentRecoveryTest, FileUtilsTest, TestSubQueryTransformerDistrib, SolrPluginUtilsTest, TermVectorComponentTest, TestRestoreCore, ConcurrentDeleteAndCreateCollectionTest, SolrCoreTest, HdfsNNFailoverTest, TestSolrQueryResponse, PreAnalyzedFieldTest, RecoveryAfterSoftCommitTest, TestPullReplica, HdfsRecoveryZkTest, CurrencyFieldTypeTest, TestLuceneMatchVersion, AssignTest, TestBulkSchemaAPI, OpenExchangeRatesOrgProviderTest, LoggingHandlerTest, BaseCdcrDistributedZkTest, TestIntervalFaceting, CollectionStateFormat2Test, DistributedQueryComponentOptimizationTest, MoveReplicaHDFSTest, TestLeaderElectionWithEmptyReplica, LeaderFailoverAfterPartitionTest, TestSolrDeletionPolicy1, DocValuesMissingTest, TermsComponentTest, TriLevelCompositeIdRoutingTest, TestRawTransformer, TestConfigSetProperties, TestStressLiveNodes, TestAddFieldRealTimeGet, TestShortCircuitedRequests, V2ApiIntegrationTest, HdfsAutoAddReplicasIntegrationTest, CoreAdminRequestStatusTest, ConjunctionSolrSpellCheckerTest, TestNumericTokenStream, TestManagedResourceStorage, CollectionsAPIDistributedZkTest, TestReload, SolrMetricReporterTest, BlockJoinFacetSimpleTest, AnalyticsQueryTest, TestCloudRecovery, TestCollationField, TestClusterProperties, TestManagedSchemaAPI, TestInPlaceUpdatesDistrib, TestStandardQParsers, DocValuesTest, InfoHandlerTest, DirectSolrSpellCheckerTest, TestSortByMinMaxFunction, UpdateLogTest, CopyFieldTest, SolrCloudExampleTest, MoreLikeThisHandlerTest, TestRuleBasedAuthorizationPlugin, TestPostingsSolrHighlighter, TestLuceneIndexBackCompat, TestCloudNestedDocsSort, TestConfigSetsAPIExclusivity, TestSolrCLIRunExample, TestExecutePlanAction, SubstringBytesRefFilterTest, TestHalfAndHalfDocValues, CloudExitableDirectoryReaderTest, TestRTGBase, TestValueSourceCache, BlobRepositoryCloudTest, BadComponentTest, TestHighFrequencyDictionaryFactory, TestFoldingMultitermQuery, UninvertDocValuesMergePolicyTest, ActionThrottleTest, HdfsBasicDistributedZkTest, TimeRoutedAliasUpdateProcessorTest, OpenCloseCoreStressTest, TestExceedMaxTermLength, StatelessScriptUpdateProcessorFactoryTest, TestSolrConfigHandler, TestAuthenticationFramework, UpdateParamsTest, LukeRequestHandlerTest, FieldMutatingUpdateProcessorTest, SimplePostToolTest, CoreSorterTest, LeaderInitiatedRecoveryOnCommitTest, JSONWriterTest, TestCloudInspectUtil, TestConfig, BigEndianAscendingWordDeserializerTest, TestObjectReleaseTracker, TestConfigOverlay, LargeFieldTest, TestExportWriter, TestExtendedDismaxParser, SolrShardReporterTest, DataDrivenBlockJoinTest, SpellingQueryConverterTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestAnalyzedSuggestions, TestSmileRequest, TestGraphMLResponseWriter, TestSolrConfigHandlerConcurrent, CustomHighlightComponentTest, MBeansHandlerTest, DocValuesMultiTest, TestConfigsApi, SolrCoreMetricManagerTest, TestBulkSchemaConcurrent, JsonLoaderTest, TestSubQueryTransformer, TestLRUStatsCache, TriggerIntegrationTest, TestManagedStopFilterFactory, TestUtilizeNode, TestNodeLostTrigger, TestQueryUtils, CdcrBootstrapTest, ConnectionManagerTest, TestRandomFaceting, TestCollectionsAPIViaSolrCloudCluster, TestLegacyFieldReuse, TestPerFieldSimilarityWithDefaultOverride, TestStressCloudBlindAtomicUpdates, TestDocumentBuilder, ZkStateWriterTest, UtilsToolTest, FullSolrCloudDistribCmdsTest, BlockJoinFacetDistribTest, DistributedFacetPivotSmallAdvancedTest, TestDelegationWithHadoopAuth, QueryResultKeyTest, OverseerCollectionConfigSetProcessorTest, AutoscalingHistoryHandlerTest, TestRecoveryHdfs, TestBlobHandler, SchemaVersionSpecificBehaviorTest, DocumentBuilderTest, HdfsSyncSliceTest, OverseerRolesTest, SOLR749Test, TestSolrCloudWithKerberosAlt, TestSearchPerf, TestRemoteStreaming, TestCoreContainer, TestSolr4Spatial, StatsComponentTest, PeerSyncTest, TestIndexSearcher, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SuggesterFSTTest, CoreAdminHandlerTest, TestStressLucene, TestTrie, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, DefaultValueUpdateProcessorTest, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, TestSurroundQueryParser, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, QueryParsingTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, HighlighterConfigTest, AlternateDirectoryTest, TestSolrIndexConfig, SolrIndexConfigTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, TestCharFilters, SynonymTokenizerTest, TestXIncludeConfig, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TimeZoneUtilsTest, TestFastOutputStream, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, TestSystemIdResolver, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestLRUCache, TestUtils, ZkNodePropsTest, SystemInfoHandlerTest, UUIDFieldTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, AddReplicaTest, AliasIntegrationTest, AssignBackwardCompatibilityTest, CleanupOldIndexTest, CollectionReloadTest, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DeleteStatusTest, DistributedQueueTest, DocValuesNotIndexedTest, ForceLeaderTest, MoveReplicaTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, RemoteQueryErrorTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, TestCloudDeleteByQuery, TestCloudPivotFacet, TestCollectionAPI, TestConfigSetsAPI, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestLockTree, TestMiniSolrCloudClusterSSL, TestPrepRecovery, TestReplicaProperties, TestRequestStatusCollectionAPI, TestSkipOverseerOperations, TestSolrCloudWithSecureImpersonation] [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=A5B9F22229685486 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=Indian/Reunion -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] ERROR 0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<< [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: [junit4] > 1) Thread[id=37645, name=jetty-launcher-8147-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) [junit4] > 2) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) [junit4] > at __randomizedtesting.SeedInfo.seed([A5B9F22229685486]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=37644, name=jetty-launcher-8147-thread-2-SendThread(127.0.0.1:59478), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) [junit4] > at __randomizedtesting.SeedInfo.seed([A5B9F22229685486]:0) [junit4] Completed [571/765 (2!)] on J1 in 64.27s, 12 tests, 2 errors <<< FAILURES! [...truncated 48785 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org