Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21226/ Java: 64bit/jdk1.8.0_144 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed. FAILED: org.apache.solr.cloud.TestAuthenticationFramework.testBasics Error Message: Error from server at http://127.0.0.1:40129/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p> <hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> Stack Trace: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:40129/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p> <hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> at __randomizedtesting.SeedInfo.seed([9705E427D041289D:AADD4A0BE8AF76ED]:0) at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:550) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1013) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233) at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126) at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40129/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p> <hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:527) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more FAILED: org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction.testNodeWithMultipleReplicasLost Error Message: Stack Trace: java.util.ConcurrentModificationException at __randomizedtesting.SeedInfo.seed([9705E427D041289D:A7C505A55833C9C1]:0) at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:901) at java.util.ArrayList$Itr.next(ArrayList.java:851) at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) at sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12548 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction [junit4] 2> 847704 INFO (SUITE-TestComputePlanAction-seed#[9705E427D041289D]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestComputePlanAction_9705E427D041289D-001/init-core-data-001 [junit4] 2> 847705 WARN (SUITE-TestComputePlanAction-seed#[9705E427D041289D]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1 [junit4] 2> 847705 INFO (SUITE-TestComputePlanAction-seed#[9705E427D041289D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 847707 INFO (SUITE-TestComputePlanAction-seed#[9705E427D041289D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847719 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 847720 DEBUG (ScheduledTrigger-2957-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847721 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeWithMultipleReplicasLost [junit4] 2> 847721 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 847721 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 847722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 847722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1 [junit4] 2> 847722 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 847722 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 847723 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 847723 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847744 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847764 DEBUG (ScheduledTrigger-2960-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847784 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847804 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847825 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847825 DEBUG (simCloudManagerPool-2959-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 847826 DEBUG (simCloudManagerPool-2959-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 847826 DEBUG (simCloudManagerPool-2959-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 847826 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 847826 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847826 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 847826 DEBUG (simCloudManagerPool-2959-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 847827 DEBUG (simCloudManagerPool-2959-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 847827 DEBUG (simCloudManagerPool-2959-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 847827 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 847827 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847827 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 847845 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847865 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847885 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847905 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847926 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 847944 DEBUG (simCloudManagerPool-2959-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 847944 DEBUG (simCloudManagerPool-2959-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 847944 DEBUG (simCloudManagerPool-2959-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 847944 DEBUG (simCloudManagerPool-2959-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 847944 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 847944 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 847945 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5 [junit4] 2> 847945 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 847945 DEBUG (simCloudManagerPool-2959-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeWithMultipleReplicasLost [junit4] 2> 847946 DEBUG (ScheduledTrigger-2960-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 847959 DEBUG (simCloudManagerPool-2959-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 847962 DEBUG (simCloudManagerPool-2959-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 847965 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 847966 DEBUG (ScheduledTrigger-2960-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 847967 INFO (simCloudManagerPool-2959-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952171157 sessionWrapper.createTime 2952170905605461, this.sessionWrapper.createTime 2952170905605461 [junit4] 2> 847967 DEBUG (simCloudManagerPool-2959-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 847986 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 847986 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 848007 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 848007 DEBUG (ScheduledTrigger-2960-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 848027 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 848027 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 848033 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction -- cluster state: znodeVersion: 2 [junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> collections:{testNodeWithMultipleReplicasLost=DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/1)={ [junit4] 2> "replicationFactor":"3", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"3", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{ [junit4] 2> "shard2":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node6":{ [junit4] 2> "node_name":"127.0.0.1:10002_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node4":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node5":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"0-7fffffff", [junit4] 2> "state":"active"}, [junit4] 2> "shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10003_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10002_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node3":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-ffffffff", [junit4] 2> "state":"active"}}}} [junit4] 2> 848037 DEBUG (simCloudManagerPool-2959-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard1: {"core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10003_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"testNodeWithMultipleReplicasLost"}} [junit4] 2> 848037 DEBUG (simCloudManagerPool-2959-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard2: {"core_node6":{ [junit4] 2> "node_name":"127.0.0.1:10002_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard2", [junit4] 2> "collection":"testNodeWithMultipleReplicasLost"}} [junit4] 2> 848047 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 848047 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr [junit4] 2> 848047 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 848047 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr [junit4] 2> 848067 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 848067 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 848067 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10000_solr] [junit4] 2> 848067 DEBUG (ScheduledTrigger-2960-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a7cfcd927905fT246xpgsof8dkbs8gpt21e6igk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952175183958111, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952175183958111], [junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} [junit4] 2> 848070 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a7cfcd927905fT246xpgsof8dkbs8gpt21e6igk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952175183958111, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952175183958111], [junit4] 2> "_enqueue_time_":2952176203217861, [junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} [junit4] 2> 848070 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"a7cfcd927905fT246xpgsof8dkbs8gpt21e6igk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952175183958111, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952175183958111], [junit4] 2> "_enqueue_time_":2952176203217861, [junit4] 2> "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 848070 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 848073 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 848074 INFO (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5 [junit4] 2> 848074 INFO (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node3 [junit4] 2> 848074 INFO (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952176530 sessionWrapper.createTime 2952176457814061, this.sessionWrapper.createTime 2952176457814061 [junit4] 2> 848074 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 848076 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5 [junit4] 2> 848076 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node3 [junit4] 2> 848077 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 848077 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10001_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10001_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10001, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10000_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":980, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10000_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10000, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10002_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":980, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10002_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10002, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10003_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10003_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10003, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 848077 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Live nodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Collection testNodeWithMultipleReplicasLost state: DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/3)={ [junit4] 2> "replicationFactor":"3", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"3", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{ [junit4] 2> "shard2":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node6":{ [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6", [junit4] 2> "leader":"true", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "node_name":"127.0.0.1:10002_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node4":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node5":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"0-7fffffff", [junit4] 2> "state":"active"}, [junit4] 2> "shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1", [junit4] 2> "leader":"true", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "node_name":"127.0.0.1:10003_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10002_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node3":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-ffffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeWithMultipleReplicasLost [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 848078 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 848079 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0 [junit4] 2> 848079 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 848079 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10000_solr [junit4] 2> 848079 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [testNodeWithMultipleReplicasLost] [junit4] 2> 848080 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 2 [junit4] 2> 848080 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 1 [junit4] 2> 848080 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 4 [junit4] 2> 848080 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testNodeWithMultipleReplicasLost 4 [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 4 [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1 [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 3 [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 848081 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 848082 DEBUG (AutoscalingActionExecutor-2961-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 571 ms for event id=a7cfcd927905fT246xpgsof8dkbs8gpt21e6igk [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestComputePlanAction -Dtests.method=testNodeWithMultipleReplicasLost -Dtests.seed=9705E427D041289D -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr-BA -Dtests.timezone=Etc/GMT+8 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII [junit4] ERROR 0.36s J0 | TestComputePlanAction.testNodeWithMultipleReplicasLost <<< [junit4] > Throwable #1: java.util.ConcurrentModificationException [junit4] > at __randomizedtesting.SeedInfo.seed([9705E427D041289D:A7C505A55833C9C1]:0) [junit4] > at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:901) [junit4] > at java.util.ArrayList$Itr.next(ArrayList.java:851) [junit4] > at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] 2> 848088 DEBUG (ScheduledTrigger-2960-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 848090 DEBUG (ScheduledTrigger-2960-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 848091 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost [junit4] 2> 848092 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 848092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 848092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 848092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 848092 INFO (simCloudManagerPool-2959-thread-17) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2152ms [junit4] 2> 848092 INFO (simCloudManagerPool-2959-thread-18) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2146ms [junit4] 2> 848092 INFO (simCloudManagerPool-2959-thread-19) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 2137ms [junit4] 2> 848092 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr [junit4] 2> 848092 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 848092 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 848093 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 848093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 848093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 848093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1 [junit4] 2> 848093 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 848094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 848094 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848114 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848135 DEBUG (ScheduledTrigger-2963-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848155 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848175 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848194 DEBUG (simCloudManagerPool-2962-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848194 DEBUG (simCloudManagerPool-2962-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 848194 DEBUG (simCloudManagerPool-2962-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848194 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 848195 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848195 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 848195 DEBUG (simCloudManagerPool-2962-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848195 DEBUG (simCloudManagerPool-2962-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 848195 DEBUG (simCloudManagerPool-2962-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848195 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9 [junit4] 2> 848195 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848195 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9 [junit4] 2> 848195 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848215 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848236 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848256 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848276 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848296 DEBUG (simCloudManagerPool-2962-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848296 DEBUG (simCloudManagerPool-2962-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 848296 DEBUG (simCloudManagerPool-2962-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr] [junit4] 2> 848296 DEBUG (simCloudManagerPool-2962-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10004_solr] [junit4] 2> 848296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 848296 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 848296 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848296 DEBUG (simCloudManagerPool-2962-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeLost [junit4] 2> 848307 DEBUG (simCloudManagerPool-2962-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 848308 DEBUG (simCloudManagerPool-2962-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 848309 INFO (simCloudManagerPool-2962-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952188278 sessionWrapper.createTime 2952188232047161, this.sessionWrapper.createTime 2952188232047161 [junit4] 2> 848309 DEBUG (simCloudManagerPool-2962-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 848316 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848317 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848333 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction -- cluster state: znodeVersion: 8 [junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10004_solr] [junit4] 2> collections:{testNodeLost=DocCollection(testNodeLost//clusterstate.json/7)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10004_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}}} [junit4] 2> 848336 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction Stopped_node : 127.0.0.1:10004_solr [junit4] 2> 848337 DEBUG (ScheduledTrigger-2963-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848337 DEBUG (simCloudManagerPool-2962-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: {"core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n2", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"testNodeLost"}} [junit4] 2> 848337 DEBUG (ScheduledTrigger-2963-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr [junit4] 2> 848337 DEBUG (ScheduledTrigger-2963-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848337 DEBUG (ScheduledTrigger-2963-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10004_solr [junit4] 2> 848357 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848357 DEBUG (ScheduledTrigger-2963-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848377 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848377 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848398 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848398 DEBUG (ScheduledTrigger-2963-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848418 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848418 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848438 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848438 DEBUG (ScheduledTrigger-2963-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848458 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848458 DEBUG (ScheduledTrigger-2963-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848478 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848478 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 848478 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10004_solr] [junit4] 2> 848479 DEBUG (ScheduledTrigger-2963-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a7d00390084c5T246xpgsof8dkbs8gpt21e6igp", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952189676913861, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952189676913861], [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 848481 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a7d00390084c5T246xpgsof8dkbs8gpt21e6igp", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952189676913861, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952189676913861], [junit4] 2> "_enqueue_time_":2952196756341861, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 848481 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"a7d00390084c5T246xpgsof8dkbs8gpt21e6igp", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2952189676913861, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952189676913861], [junit4] 2> "_enqueue_time_":2952196756341861, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 848481 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 848486 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 848487 INFO (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10005_solr&inPlaceMove=true&replica=core_node1 [junit4] 2> 848487 INFO (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952197157 sessionWrapper.createTime 2952197138904111, this.sessionWrapper.createTime 2952197138904111 [junit4] 2> 848487 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 848487 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 848487 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10001_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":980, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10001_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10001, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10004_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10004_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10004, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10005_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":0, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":1000, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10005_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10005, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 848487 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Live nodes: [127.0.0.1:10001_solr] [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Collection testNodeLost state: DocCollection(testNodeLost//clusterstate.json/9)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10004_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "core":"testNodeLost_shard1_replica_n2", [junit4] 2> "leader":"true", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT"}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 2 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10004_solr [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10005_solr [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [testNodeLost] [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testNodeLost 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 3 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 848488 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 848489 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger [junit4] 2> 848489 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - BEFORE_ACTION 1 [junit4] 2> 848489 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 848508 DEBUG (AutoscalingActionExecutor-2964-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 1337 ms for event id=a7d00390084c5T246xpgsof8dkbs8gpt21e6igp [junit4] 2> 848509 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded [junit4] 2> 848509 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 848509 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10004_solr [junit4] 2> 848509 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10005_solr [junit4] 2> 848509 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 848509 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 848509 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion -1 [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 848514 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848534 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848555 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848575 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848595 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848615 DEBUG (simCloudManagerPool-2965-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848615 DEBUG (simCloudManagerPool-2965-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 848615 DEBUG (simCloudManagerPool-2965-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 848615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 848616 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848616 DEBUG (simCloudManagerPool-2965-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848616 DEBUG (simCloudManagerPool-2965-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 848616 DEBUG (simCloudManagerPool-2965-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 848616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 848616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 848636 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848656 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848676 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848696 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848717 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848718 DEBUG (simCloudManagerPool-2965-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848719 DEBUG (simCloudManagerPool-2965-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 848719 DEBUG (simCloudManagerPool-2965-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848719 DEBUG (simCloudManagerPool-2965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848719 DEBUG (simCloudManagerPool-2965-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 848720 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 848720 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15 [junit4] 2> 848720 DEBUG (ScheduledTrigger-2966-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 848720 DEBUG (ScheduledTrigger-2966-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1 [junit4] 2> 848724 DEBUG (simCloudManagerPool-2965-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848724 DEBUG (simCloudManagerPool-2965-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 848725 DEBUG (simCloudManagerPool-2965-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848725 DEBUG (simCloudManagerPool-2965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848725 DEBUG (simCloudManagerPool-2965-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 848725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 848725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 848725 DEBUG (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeAdded [junit4] 2> 848736 WARN (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 2 on collection testNodeAdded is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance. [junit4] 2> 848737 DEBUG (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 848737 DEBUG (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 848737 INFO (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952209685 sessionWrapper.createTime 2952209671203361, this.sessionWrapper.createTime 2952209671203361 [junit4] 2> 848737 DEBUG (simCloudManagerPool-2965-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 848737 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848741 DEBUG (ScheduledTrigger-2966-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 848741 DEBUG (ScheduledTrigger-2966-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1 [junit4] 2> 848758 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 848761 DEBUG (simCloudManagerPool-2965-thread-8) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 848761 DEBUG (simCloudManagerPool-2965-thread-8) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 848761 DEBUG (simCloudManagerPool-2965-thread-8) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848761 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 848761 DEBUG (simCloudManagerPool-2965-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr] [junit4] 2> 848761 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1 [junit4] 2> 848761 DEBUG (simCloudManagerPool-2965-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 848761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 848761 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17 [junit4] 2> 848778 DEBUG (ScheduledTrigger-2966-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848781 DEBUG (ScheduledTrigger-2966-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 848781 DEBUG (ScheduledTrigger-2966-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 848781 DEBUG (ScheduledTrigger-2966-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10006_solr at time 2952211894223561 [junit4] 2> 848798 DEBUG (ScheduledTrigger-2966-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 848802 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 848802 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 848802 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10006_solr] added at times [2952211894223561], now=2952212911316211 [junit4] 2> 848802 DEBUG (ScheduledTrigger-2966-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"a7d056541c2c9T246xpgsof8dkbs8gpt21e6igu", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2952211894223561, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952211894223561], [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 848802 DEBUG (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"a7d056541c2c9T246xpgsof8dkbs8gpt21e6igu", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2952211894223561, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952211894223561], [junit4] 2> "_enqueue_time_":2952212918273861, [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 848803 DEBUG (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"a7d056541c2c9T246xpgsof8dkbs8gpt21e6igu", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2952211894223561, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2952211894223561], [junit4] 2> "_enqueue_time_":2952212918273861, [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 848803 DEBUG (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 848803 DEBUG (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 848803 INFO (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeAdded&targetNode=127.0.0.1:10006_solr&inPlaceMove=true&replica=core_node1 [junit4] 2> 848803 INFO (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2952212997 sessionWrapper.createTime 2952212982508761, this.sessionWrapper.createTime 2952212982508761 [junit4] 2> 848803 DEBUG (AutoscalingActionExecutor-2967-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 848804 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction Node values: { [junit4] 2> "127.0.0.1:10001_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":960, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10001_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10001, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10006_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":0, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":1000, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10006_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10006, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 848804 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction Live nodes: [127.0.0.1:10001_solr, 127.0.0.1:10006_solr], collection state: DocCollection(testNodeAdded//clusterstate.json/11)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 848804 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 848804 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[9705E427D041289D]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10001_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":960, [junit4] 2> "sysprop.java.version":"1.8.0_144", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10001 [...truncated too long message...] Executor-7526-thread-1) [n:127.0.0.1:35901_solr c:testcollection s:shard2 r:core_node8 x:testcollection_shard2_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard2.leader, tag=539969667 [junit4] 2> 1981022 INFO (coreCloseExecutor-7526-thread-2) [n:127.0.0.1:35901_solr c:testcollection s:shard1 r:core_node5 x:testcollection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard1.replica_n2, tag=1194525713 [junit4] 2> 1981022 INFO (coreCloseExecutor-7526-thread-2) [n:127.0.0.1:35901_solr c:testcollection s:shard1 r:core_node5 x:testcollection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17f72e19: rootName = solr_35901, domain = solr.core.testcollection.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.testcollection.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@1dac566b [junit4] 2> 1981026 INFO (coreCloseExecutor-7525-thread-1) [n:127.0.0.1:40129_solr c:testcollection s:shard2 r:core_node7 x:testcollection_shard2_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard2.leader, tag=1721056170 [junit4] 2> 1981026 INFO (coreCloseExecutor-7525-thread-2) [n:127.0.0.1:40129_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1640779879 [junit4] 2> 1981027 INFO (jetty-closer-3684-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21cdc0f0{/solr,null,UNAVAILABLE} [junit4] 2> 1981031 INFO (coreCloseExecutor-7526-thread-2) [n:127.0.0.1:35901_solr c:testcollection s:shard1 r:core_node5 x:testcollection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1194525713 [junit4] 2> 1981032 INFO (jetty-closer-3684-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99303564261851142-127.0.0.1:35901_solr-n_0000000000) closing [junit4] 2> 1981032 INFO (OverseerStateUpdate-99303564261851142-127.0.0.1:35901_solr-n_0000000000) [n:127.0.0.1:35901_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35901_solr [junit4] 2> 1981033 WARN (OverseerAutoScalingTriggerThread-99303564261851142-127.0.0.1:35901_solr-n_0000000000) [n:127.0.0.1:35901_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1981035 INFO (jetty-closer-3684-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@781726e7{/solr,null,UNAVAILABLE} [junit4] 2> 1981035 ERROR (SUITE-TestAuthenticationFramework-seed#[9705E427D041289D]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 1981036 INFO (SUITE-TestAuthenticationFramework-seed#[9705E427D041289D]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34101 34101 [junit4] 2> 1981511 INFO (Thread-5689) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34101 34101 [junit4] 2> 1981512 WARN (Thread-5689) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 4 /solr/configs/solrCloudCollectionConfig [junit4] 2> 3 /solr/aliases.json [junit4] 2> 3 /solr/clusterprops.json [junit4] 2> 2 /solr/security.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 19 /solr/collections/testcollection/state.json [junit4] 2> 3 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 3 /solr/live_nodes [junit4] 2> 3 /solr/collections [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestAuthenticationFramework_9705E427D041289D-001 [junit4] 2> Jan 06, 2018 3:22:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=PostingsFormat(name=LuceneVarGapFixedInterval), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=222, maxMBSortInHeap=7.473316188855817, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@7008c249), locale=nn-NO, timezone=Australia/Victoria [junit4] 2> NOTE: Linux 4.10.0-40-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=1,free=248944624,total=518979584 [junit4] 2> NOTE: All tests run in this JVM: [TestPerFieldSimilarityWithDefaultOverride, HdfsRestartWhileUpdatingTest, RestartWhileUpdatingTest, TestDistribIDF, TestCloudPseudoReturnFields, ZkSolrClientTest, TestFieldCacheSortRandom, TestStressLucene, TestBM25SimilarityFactory, TestManagedSchemaThreadSafety, LeaderInitiatedRecoveryOnShardRestartTest, TestUninvertingReader, TestPolicyCloud, DeleteShardTest, TestRemoteStreaming, TestLegacyFieldCache, DirectUpdateHandlerTest, CloudExitableDirectoryReaderTest, SolrIndexMetricsTest, TestClassicSimilarityFactory, TestHalfAndHalfDocValues, SearchRateTriggerTest, NotRequiredUniqueKeyTest, TestLegacyField, DistributedFacetPivotSmallTest, PreAnalyzedFieldTest, AddReplicaTest, UninvertDocValuesMergePolicyTest, TestCoreBackup, TestNRTOpen, SearchHandlerTest, DistributedVersionInfoTest, ShardRoutingTest, MetricTriggerTest, TestAnalyzedSuggestions, TestDocumentBuilder, JvmMetricsTest, StatsReloadRaceTest, TestImplicitCoreProperties, SampleTest, NumericFieldsTest, TestDistribStateManager, SynonymTokenizerTest, SpatialHeatmapFacetsTest, CdcrReplicationDistributedZkTest, TestFieldSortValues, CdcrUpdateLogTest, LukeRequestHandlerTest, TestNestedDocsSort, OutOfBoxZkACLAndCredentialsProvidersTest, DistributedQueryComponentOptimizationTest, TestPayloadScoreQParserPlugin, QueryEqualityTest, TestCorePropertiesReload, TestPerFieldSimilarity, TestPivotHelperCode, MissingSegmentRecoveryTest, TestNumericRangeQuery64, CoreMergeIndexesAdminHandlerTest, TestMaxTokenLenTokenizer, TestLRUStatsCache, TestRebalanceLeaders, TestRequestStatusCollectionAPI, TestSchemaVersionResource, TestMaxScoreQueryParser, TestUtilizeNode, SpellingQueryConverterTest, TestFieldResource, BlockJoinFacetSimpleTest, BaseCdcrDistributedZkTest, TestSQLHandlerNonCloud, TestNamedUpdateProcessors, AnalyticsMergeStrategyTest, DistribDocExpirationUpdateProcessorTest, DistributedIntervalFacetingTest, OpenCloseCoreStressTest, AutoscalingHistoryHandlerTest, HdfsChaosMonkeyNothingIsSafeTest, TestConfigSetImmutable, UUIDUpdateProcessorFallbackTest, TestDistributedStatsComponentCardinality, SuggesterFSTTest, TestReqParamsAPI, TestSchemaNameResource, DistributedTermsComponentTest, TestFileDictionaryLookup, ShardRoutingCustomTest, TestImpersonationWithHadoopAuth, TestMinMaxOnMultiValuedField, TestJavabinTupleStreamParser, ClassificationUpdateProcessorIntegrationTest, DistributedDebugComponentTest, CollectionTooManyReplicasTest, TestNoOpRegenerator, UpdateLogTest, TestComplexPhraseLeadingWildcard, TestShortCircuitedRequests, SpellCheckCollatorWithCollapseTest, DocValuesMultiTest, RulesTest, CdcrVersionReplicationTest, TestFieldCache, PropertiesRequestHandlerTest, PolyFieldTest, MoveReplicaTest, BlockCacheTest, DebugComponentTest, SolrShardReporterTest, HighlighterMaxOffsetTest, SimpleCollectionCreateDeleteTest, TestAtomicUpdateErrorCases, DOMUtilTest, TestStressRecovery, TestSimpleTrackingShardHandler, SolrXmlInZkTest, MBeansHandlerTest, DistributedFacetSimpleRefinementLongTailTest, PingRequestHandlerTest, LargeFieldTest, TestDocSet, OpenExchangeRatesOrgProviderTest, SchemaVersionSpecificBehaviorTest, TestMacroExpander, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, BasicZkTest, ClusterStateUpdateTest, TestRandomFaceting, TestRandomDVFaceting, ZkCLITest, TestDistributedGrouping, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestStressReorder, TestJoin, TestReload, TestStressVersions, TestRangeQuery, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, SuggesterTSTTest, TestTrie, NoCacheHeaderTest, TestUpdate, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, CacheHeaderTest, TestWriterPerf, PrimitiveFieldTypeTest, TestOmitPositions, PathHierarchyTokenizerFactoryTest, TestSolrQueryParser, SolrPluginUtilsTest, TestCollationField, TestCSVResponseWriter, TestPHPSerializedResponseWriter, HighlighterConfigTest, SOLR749Test, TestQuerySenderNoQuery, ResponseLogComponentTest, TestMergePolicyConfig, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, SpellPossibilityIteratorTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestFastWriter, ScriptEngineTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, AssignTest, AsyncCallRequestStatusResponseTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionStateFormat2Test, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DistribCursorPagingTest, HealthCheckHandlerTest, HttpPartitionTest, LegacyCloudClusterPropTest, MoveReplicaHDFSTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework] [junit4] Completed [510/766 (2!)] on J1 in 9.95s, 1 test, 1 error <<< FAILURES! [...truncated 48979 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
