Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2278/
7 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.analytics.OverallAnalyticsTest Error Message: 1 thread leaked from SUITE scope at org.apache.solr.analytics.OverallAnalyticsTest: 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.analytics.OverallAnalyticsTest: 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([F7449FA859D9CE6B]:0) FAILED: junit.framework.TestSuite.org.apache.solr.analytics.OverallAnalyticsTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([F7449FA859D9CE6B]:0) FAILED: org.apache.solr.cloud.TestTlogReplica.testRecovery Error Message: Can not find doc 8 in https://127.0.0.1:44378/solr Stack Trace: java.lang.AssertionError: Can not find doc 8 in https://127.0.0.1:44378/solr at __randomizedtesting.SeedInfo.seed([8CCFA3740F1B19D6:4D3FDAD8224BD371]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNotNull(Assert.java:526) at org.apache.solr.cloud.TestTlogReplica.checkRTG(TestTlogReplica.java:885) at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:599) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testCooldown Error Message: Stack Trace: java.lang.AssertionError at __randomizedtesting.SeedInfo.seed([8CCFA3740F1B19D6:BD71CE9071B16C24]:0) at org.junit.Assert.fail(Assert.java:92) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertTrue(Assert.java:54) at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testCooldown(TriggerIntegrationTest.java:1177) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate Error Message: The trigger did not fire at all Stack Trace: java.lang.AssertionError: The trigger did not fire at all at __randomizedtesting.SeedInfo.seed([8CCFA3740F1B19D6:D187BDFDC0DDBF99]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.handler.TestSystemCollAutoCreate.testAutoCreate Error Message: Could not load collection from ZK: .system Stack Trace: org.apache.solr.common.SolrException: Could not load collection from ZK: .system at __randomizedtesting.SeedInfo.seed([8CCFA3740F1B19D6:B39417A5B965D0BB]:0) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1108) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647) at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:137) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:117) at org.apache.solr.handler.TestSystemCollAutoCreate.testAutoCreate(TestSystemCollAutoCreate.java:27) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/.system/state.json at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1120) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1106) ... 45 more FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory Error Message: expected:<5> but was:<0> Stack Trace: java.lang.AssertionError: expected:<5> but was:<0> at __randomizedtesting.SeedInfo.seed([8CCFA3740F1B19D6:E1330789B553E6D1]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.failNotEquals(Assert.java:647) at org.junit.Assert.assertEquals(Assert.java:128) at org.junit.Assert.assertEquals(Assert.java:472) at org.junit.Assert.assertEquals(Assert.java:456) at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 11735 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> 156694 INFO (SUITE-TestTriggerIntegration-seed#[8CCFA3740F1B19D6]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_8CCFA3740F1B19D6-001/init-core-data-001 [junit4] 2> 156695 WARN (SUITE-TestTriggerIntegration-seed#[8CCFA3740F1B19D6]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=81 numCloses=81 [junit4] 2> 156695 INFO (SUITE-TestTriggerIntegration-seed#[8CCFA3740F1B19D6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 156729 INFO (SUITE-TestTriggerIntegration-seed#[8CCFA3740F1B19D6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 156745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 156745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 156745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [] [junit4] 2> 156745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 156745 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 156757 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 156757 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 156810 DEBUG (ScheduledTrigger-278-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 156824 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger [junit4] 2> 156824 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 156824 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 156824 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 156834 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 156844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 156845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 156845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10010_solr, 127.0.0.1:10009_solr] [junit4] 2> 156845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 156845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 156858 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 156858 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 156874 DEBUG (ScheduledTrigger-281-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 156907 DEBUG (ScheduledTrigger-281-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157022 DEBUG (ScheduledTrigger-281-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157055 DEBUG (ScheduledTrigger-281-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157075 DEBUG (ScheduledTrigger-281-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157091 DEBUG (simCloudManagerPool-280-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157091 DEBUG (simCloudManagerPool-280-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 157091 DEBUG (simCloudManagerPool-280-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10010_solr, 127.0.0.1:10009_solr] [junit4] 2> 157091 DEBUG (simCloudManagerPool-280-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10010_solr, 127.0.0.1:10009_solr] [junit4] 2> 157092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 157092 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 157098 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 157098 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10010_solr [junit4] 2> 157098 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 157098 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10010_solr [junit4] 2> 157118 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 [junit4] 2> 157118 DEBUG (ScheduledTrigger-281-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 157118 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10010_solr] [junit4] 2> 157118 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e1f4b94b39dT8v1uqefexip0ehhtpxxf0hxcy", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":1493271202542493, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493271202542493], [junit4] 2> "nodeNames":["127.0.0.1:10010_solr"]}} [junit4] 2> 157119 DEBUG (ScheduledTrigger-281-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger] [junit4] 2> 157122 DEBUG (AutoscalingActionExecutor-282-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e1f4b94b39dT8v1uqefexip0ehhtpxxf0hxcy", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":1493271202542493, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493271202542493], [junit4] 2> "_enqueue_time_":1493272261695993, [junit4] 2> "nodeNames":["127.0.0.1:10010_solr"]}} [junit4] 2> 157125 DEBUG (AutoscalingActionExecutor-282-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157125 DEBUG (AutoscalingActionExecutor-282-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms [junit4] 2> 157125 DEBUG (AutoscalingActionExecutor-282-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 168 ms for event id=54e1f4b94b39dT8v1uqefexip0ehhtpxxf0hxcy [junit4] 2> 157125 DEBUG (simCloudManagerPool-280-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157125 DEBUG (simCloudManagerPool-280-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 157125 DEBUG (simCloudManagerPool-280-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr] [junit4] 2> 157125 DEBUG (simCloudManagerPool-280-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr] [junit4] 2> 157138 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 157138 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157138 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 1 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 1 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10010_solr [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 157174 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 157175 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger [junit4] 2> 157175 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 157175 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 157179 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling [junit4] 2> 157179 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 157179 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 157179 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 157179 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157179 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5 [junit4] 2> 157180 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10010_solr [junit4] 2> 157180 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 157180 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 157180 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 157180 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157181 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 157181 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157210 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157247 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157267 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157282 DEBUG (simCloudManagerPool-283-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157282 DEBUG (simCloudManagerPool-283-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 157288 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157292 DEBUG (simCloudManagerPool-283-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157295 DEBUG (simCloudManagerPool-283-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157295 DEBUG (simCloudManagerPool-283-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157298 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 157298 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157312 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 157336 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157336 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157336 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157353 DEBUG (simCloudManagerPool-283-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157356 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157369 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157369 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 157377 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157385 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 157385 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157385 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 157385 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 157385 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 157385 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 1493285566783593 [junit4] 2> 157385 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10012_solr] added at times [1493285566783593], now=1493285571339043 [junit4] 2> 157385 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e22a3c1a069T8v1uqefexip0ehhtpxxf0hxd1", [junit4] 2> "source":"node_added_trigger2", [junit4] 2> "eventTime":1493285566783593, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493285566783593], [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157386 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1] [junit4] 2> 157417 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e22a3c1a069T8v1uqefexip0ehhtpxxf0hxd1", [junit4] 2> "source":"node_added_trigger2", [junit4] 2> "eventTime":1493285566783593, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493285566783593], [junit4] 2> "_enqueue_time_":1493285594985843, [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157418 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2 [junit4] 2> 157418 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157418 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 157418 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 157418 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 33 ms for event id=54e22a3c1a069T8v1uqefexip0ehhtpxxf0hxd1 [junit4] 2> 157518 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 157518 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157519 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 157519 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 1493292240273593 [junit4] 2> 157519 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10012_solr] added at times [1493292240273593], now=1493292241905393 [junit4] 2> 157519 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e24318700b9T8v1uqefexip0ehhtpxxf0hxd4", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493292240273593, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493292240273593], [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157519 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1] [junit4] 2> 157543 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 157543 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 157555 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e24318700b9T8v1uqefexip0ehhtpxxf0hxd4", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493292240273593, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493292240273593], [junit4] 2> "_enqueue_time_":1493292250966793, [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157555 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration last action at 1493287190946143 time = 1493294059946493 [junit4] 2> 157555 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1 [junit4] 2> 157555 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157556 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 157556 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 157556 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 39 ms for event id=54e24318700b9T8v1uqefexip0ehhtpxxf0hxd4 [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157556 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157557 DEBUG (simCloudManagerPool-283-thread-7) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157557 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157557 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 157557 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157557 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157558 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157558 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157558 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 157558 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157558 DEBUG (simCloudManagerPool-283-thread-8) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10012_solr, 127.0.0.1:10011_solr] [junit4] 2> 157567 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 157567 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157567 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157567 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3 [junit4] 2> 157567 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 157567 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 2 [junit4] 2> 157568 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10012_solr [junit4] 2> 157568 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10012_solr] [junit4] 2> 157568 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e24c38cb03fT8v1uqefexip0ehhtpxxf0hxd7", [junit4] 2> "source":"node_lost_trigger2", [junit4] 2> "eventTime":1493294690119743, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493294690119743], [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157568 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, .auto_add_replicas, node_added_trigger2, node_lost_trigger1, node_added_trigger1] [junit4] 2> 157568 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e24c38cb03fT8v1uqefexip0ehhtpxxf0hxd7", [junit4] 2> "source":"node_lost_trigger2", [junit4] 2> "eventTime":1493294690119743, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493294690119743], [junit4] 2> "_enqueue_time_":1493294698120743, [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157568 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger2 [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 157569 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 28 ms for event id=54e24c38cb03fT8v1uqefexip0ehhtpxxf0hxd7 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157669 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10012_solr [junit4] 2> 157670 DEBUG (ScheduledTrigger-284-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10012_solr [junit4] 2> 157670 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10012_solr] [junit4] 2> 157670 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e25f392aef5T8v1uqefexip0ehhtpxxf0hxda", [junit4] 2> "source":"node_lost_trigger1", [junit4] 2> "eventTime":1493299790786293, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493299790786293], [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157670 DEBUG (ScheduledTrigger-284-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_lost_trigger2, .auto_add_replicas, node_added_trigger2, node_lost_trigger1, node_added_trigger1] [junit4] 2> 157670 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e25f392aef5T8v1uqefexip0ehhtpxxf0hxda", [junit4] 2> "source":"node_lost_trigger1", [junit4] 2> "eventTime":1493299790786293, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493299790786293], [junit4] 2> "_enqueue_time_":1493299800997143, [junit4] 2> "nodeNames":["127.0.0.1:10012_solr"]}} [junit4] 2> 157670 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration last action at 1493294729649743 time = 1493299824893643 [junit4] 2> 157670 INFO (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_lost_trigger1 [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testTriggerThrottling [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 157670 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10012_solr [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger2 after 100ms [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 4 [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 8 [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger1 after 100ms [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1 [junit4] 2> 157671 DEBUG (AutoscalingActionExecutor-285-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 33 ms for event id=54e25f392aef5T8v1uqefexip0ehhtpxxf0hxda [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger2 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger2 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 157671 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 157675 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue [junit4] 2> 157675 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 157675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 157675 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10012_solr [junit4] 2> 157675 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 157675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 157675 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157675 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 157675 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) [junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion -1 [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 157676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 157676 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157697 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157717 DEBUG (ScheduledTrigger-287-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157737 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157758 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157777 DEBUG (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 157777 DEBUG (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 157778 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157778 DEBUG (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157778 INFO (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 157778 DEBUG (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10011_solr] [junit4] 2> 157778 DEBUG (simCloudManagerPool-286-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 157778 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 157779 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157779 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 157779 DEBUG (ScheduledTrigger-287-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157779 DEBUG (ScheduledTrigger-287-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 157779 DEBUG (ScheduledTrigger-287-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10013_solr at time 1493305265889443 [junit4] 2> 157798 DEBUG (ScheduledTrigger-287-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 157799 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157800 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 157800 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10013_solr] added at times [1493305265889443], now=1493306290250093 [junit4] 2> 157800 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"]}} [junit4] 2> 157800 DEBUG (ScheduledTrigger-287-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 157800 DEBUG (AutoscalingActionExecutor-288-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "_enqueue_time_":1493306304679843, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"]}} [junit4] 2> 157800 INFO (AutoscalingActionExecutor-288-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "_enqueue_time_":1493306304679843, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"]}} [junit4] 2> 157801 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 157801 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 157801 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 157801 WARN (AutoscalingActionExecutor-288-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 157801 DEBUG (AutoscalingActionExecutor-288-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 34 ms for event id=54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10013_solr] [junit4] 2> 157802 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 127.0.0.1:10013_solr] [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1 [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10011_solr [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 157802 DEBUG (ScheduledTrigger-290-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 157802 DEBUG (ScheduledTrigger-290-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr [junit4] 2> 157802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 157803 DEBUG (ScheduledTrigger-290-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "_dequeue_time_":1493306436661193, [junit4] 2> "_enqueue_time_":1493306304679843, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 157803 DEBUG (ScheduledTrigger-290-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 157803 DEBUG (AutoscalingActionExecutor-291-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "_dequeue_time_":1493306436661193, [junit4] 2> "_enqueue_time_":1493306304679843, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 157803 DEBUG (ScheduledTrigger-290-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 157803 INFO (AutoscalingActionExecutor-291-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1493305265889443, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1493305265889443], [junit4] 2> "_dequeue_time_":1493306436661193, [junit4] 2> "_enqueue_time_":1493306304679843, [junit4] 2> "nodeNames":["127.0.0.1:10013_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 157803 DEBUG (ScheduledTrigger-290-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 157805 DEBUG (AutoscalingActionExecutor-291-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 157805 DEBUG (AutoscalingActionExecutor-291-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 157805 DEBUG (AutoscalingActionExecutor-291-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 86 ms for event id=54e2739ea20a3T8v1uqefexip0ehhtpxxf0hxdd [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 157901 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10011_solr [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 157902 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[8CCFA3740F1B19D6]) [ ] o.a.s.c. [...truncated too long message...] t, TestCoreDiscovery, SpellCheckCollatorTest, TestFoldingMultitermQuery, SuggesterWFSTTest, TestCSVLoader, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DocValuesMultiTest, XsltUpdateRequestHandlerTest, LukeRequestHandlerTest, TestQueryTypes, TermsComponentTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, TestJmxIntegration, MBeansHandlerTest, SampleTest, TestDocSet, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastWriter, TestFastLRUCache, DateMathParserTest, DateFieldTest, TestDocumentBuilder, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, TestRTGBase, DistributedIntervalFacetingTest, TestTolerantSearch, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, ConnectionReuseTest, CloudExitableDirectoryReaderTest, ConcurrentCreateRoutedAliasTest, ConfigSetsAPITest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteNodeTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, NodeMutatorTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerRolesTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, SaslZkACLProviderTest, SolrXmlInZkTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPivotFacet, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestPrepRecovery, TestShortCircuitedRequests, TestTlogReplica, ConcurrentDeleteAndCreateCollectionTest, HdfsCollectionsAPIDistributedZkTest, TestCollectionAPI, AutoScalingHandlerTest, MetricTriggerTest, TriggerIntegrationTest] [junit4] Completed [613/770 (5!)] on J1 in 195.94s, 14 tests, 1 failure <<< FAILURES! [...truncated 3739 lines...] [junit4] Suite: org.apache.solr.analytics.OverallAnalyticsTest [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/contrib/solr-analytics/test/J1/temp/solr.analytics.OverallAnalyticsTest_F7449FA859D9CE6B-001/init-core-data-001 [junit4] 2> Jan 27, 2018 6:54:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s). [junit4] 2> Jan 27, 2018 6:54:53 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.analytics.OverallAnalyticsTest: [junit4] 2> 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] [junit4] 2> at sun.misc.Unsafe.park(Native Method) [junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) [junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) [junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Jan 27, 2018 6:54:53 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll [junit4] 2> INFO: Starting to interrupt leaked threads: [junit4] 2> 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] [junit4] 2> Jan 27, 2018 6:54:56 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll [junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated: [junit4] 2> 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] [junit4] 2> at sun.misc.Unsafe.park(Native Method) [junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) [junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) [junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {int_i=PostingsFormat(name=LuceneVarGapDocFreqInterval), double_dm=Lucene50(blocksize=128), float_fm=PostingsFormat(name=MockRandom), int_im=PostingsFormat(name=MockRandom), date_dt=PostingsFormat(name=LuceneVarGapDocFreqInterval), date_dtm=PostingsFormat(name=Memory), long_lm=PostingsFormat(name=Memory), double_d=PostingsFormat(name=MockRandom), float_f=Lucene50(blocksize=128), long_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=Memory), string_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=Lucene50(blocksize=128), string_sm=PostingsFormat(name=MockRandom)}, docValues:{int_i=DocValuesFormat(name=Memory), double_dm=DocValuesFormat(name=Lucene70), float_fm=DocValuesFormat(name=Direct), int_im=DocValuesFormat(name=Direct), date_dt=DocValuesFormat(name=Memory), date_dtm=DocValuesFormat(name=Asserting), long_lm=DocValuesFormat(name=Asserting), double_d=DocValuesFormat(name=Direct), float_f=DocValuesFormat(name=Lucene70), long_l=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Asserting), string_s=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Lucene70), string_sm=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=916, maxMBSortInHeap=6.895288436638899, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2291dc25), locale=ar-EG, timezone=Europe/London [junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=245170896,total=331874304 [junit4] 2> NOTE: All tests run in this JVM: [LegacyRangeFacetTest, AddFunctionTest, CastingStringValueTest, ExistsFunctionTest, LegacyQueryFacetCloudTest, CastingFloatValueTest, LongFieldsTest, FloorFunctionTest, AbstractAnalyticsFieldTest, LegacyFieldFacetExtrasCloudTest, DoubleFieldsTest, CastingLongValueStreamTest, OrFunctionTest, OverallAnalyticsTest] [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverallAnalyticsTest -Dtests.seed=F7449FA859D9CE6B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar-EG -Dtests.timezone=Europe/London -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1 [junit4] ERROR 0.00s J1 | OverallAnalyticsTest (suite) <<< [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.analytics.OverallAnalyticsTest: [junit4] > 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) [junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) [junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([F7449FA859D9CE6B]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=488, name=qtp424341292-488, state=TIMED_WAITING, group=TGRP-OverallAnalyticsTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) [junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) [junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([F7449FA859D9CE6B]:0) [junit4] Completed [70/78 (1!)] on J1 in 71.19s, 1 test, 2 errors <<< FAILURES! [...truncated 44936 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org