Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21183/ Java: 64bit/jdk-9.0.1 -XX:-UseCompressedOops -XX:+UseParallelGC
3 tests failed. FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI Error Message: Error from server at http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6: ClusterState says we are the leader (http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6), but locally we don't think so. Request came from null Stack Trace: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6: ClusterState says we are the leader (http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6), but locally we don't think so. Request came from null at __randomizedtesting.SeedInfo.seed([373B76FDD4175A98:7F4E0249D224750D]:0) at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:550) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1013) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:461) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6: ClusterState says we are the leader (http://127.0.0.1:37235/solr/awhollynewcollection_0_shard2_replica_n6), but locally we don't think so. Request came from null at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:527) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) ... 1 more FAILED: org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction.testNodeLost Error Message: Stack Trace: java.util.ConcurrentModificationException at __randomizedtesting.SeedInfo.seed([373B76FDD4175A98:882EB80357FD3F1E]:0) at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:939) at java.base/java.util.ArrayList$Itr.next(ArrayList.java:893) at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication Error Message: Index 0 out-of-bounds for length 0 Stack Trace: java.lang.IndexOutOfBoundsException: Index 0 out-of-bounds for length 0 at __randomizedtesting.SeedInfo.seed([373B76FDD4175A98:23732DA8F710E786]:0) at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) at java.base/java.util.Objects.checkIndex(Objects.java:372) at java.base/java.util.ArrayList.get(ArrayList.java:440) at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Build Log: [...truncated 11886 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction [junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestComputePlanAction_373B76FDD4175A98-001/init-core-data-001 [junit4] 2> 126055 WARN (SUITE-TestComputePlanAction-seed#[373B76FDD4175A98]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11 [junit4] 2> 126058 INFO (SUITE-TestComputePlanAction-seed#[373B76FDD4175A98]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 126059 INFO (SUITE-TestComputePlanAction-seed#[373B76FDD4175A98]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0) [junit4] 2> 126059 INFO (SUITE-TestComputePlanAction-seed#[373B76FDD4175A98]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 126068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 126068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 126068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 126068 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] 2> 126069 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost [junit4] 2> 126069 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 126070 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126070 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0 [junit4] 2> 126070 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 126070 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 126070 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 126070 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 126071 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126071 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1247) [junit4] 2> at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 126071 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 126074 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126094 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126114 DEBUG (ScheduledTrigger-492-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126134 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126155 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126175 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126187 DEBUG (simCloudManagerPool-491-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126187 DEBUG (simCloudManagerPool-491-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 126188 DEBUG (simCloudManagerPool-491-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126189 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 126190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126190 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 126190 DEBUG (simCloudManagerPool-491-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126191 DEBUG (simCloudManagerPool-491-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 126191 DEBUG (simCloudManagerPool-491-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126191 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 126191 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126191 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 126195 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126215 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126236 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126256 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126276 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126295 DEBUG (simCloudManagerPool-491-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126296 DEBUG (simCloudManagerPool-491-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 126296 DEBUG (simCloudManagerPool-491-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 126296 DEBUG (simCloudManagerPool-491-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 126296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 126296 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126296 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5 [junit4] 2> 126296 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126296 DEBUG (simCloudManagerPool-491-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeLost [junit4] 2> 126309 DEBUG (simCloudManagerPool-491-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 126310 DEBUG (simCloudManagerPool-491-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 126316 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126317 DEBUG (ScheduledTrigger-492-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126318 INFO (simCloudManagerPool-491-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433317439 sessionWrapper.createTime 2433317042727861, this.sessionWrapper.createTime 2433317042727861 [junit4] 2> 126318 DEBUG (simCloudManagerPool-491-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 126337 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126337 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126342 DEBUG (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction -- cluster state: znodeVersion: 2 [junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> collections:{testNodeLost=DocCollection(testNodeLost//clusterstate.json/1)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}}} [junit4] 2> 126344 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction Stopped_node : 127.0.0.1:10001_solr [junit4] 2> 126346 DEBUG (simCloudManagerPool-491-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: {"core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n1", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"testNodeLost"}} [junit4] 2> 126357 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126357 DEBUG (ScheduledTrigger-492-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126357 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr [junit4] 2> 126357 DEBUG (ScheduledTrigger-492-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr [junit4] 2> 126377 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126377 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126397 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126397 DEBUG (ScheduledTrigger-492-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126418 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126418 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126438 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126438 DEBUG (ScheduledTrigger-492-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126458 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126458 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126478 DEBUG (ScheduledTrigger-492-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126478 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126498 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 126498 DEBUG (ScheduledTrigger-492-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126498 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr] [junit4] 2> 126499 DEBUG (ScheduledTrigger-492-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"8a51751d0a399Teyz8mtz9bcgiplhk5bgkx86jk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433319389143961, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433319389143961], [junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} [junit4] 2> 126501 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"8a51751d0a399Teyz8mtz9bcgiplhk5bgkx86jk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433319389143961, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433319389143961], [junit4] 2> "_enqueue_time_":2433326484081011, [junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} [junit4] 2> 126502 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"8a51751d0a399Teyz8mtz9bcgiplhk5bgkx86jk", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433319389143961, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433319389143961], [junit4] 2> "_enqueue_time_":2433326484081011, [junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 126502 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 126503 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 126504 INFO (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node2 [junit4] 2> 126504 INFO (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433326752 sessionWrapper.createTime 2433326675907711, this.sessionWrapper.createTime 2433326675907711 [junit4] 2> 126504 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 126505 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10001_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10001_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10001, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10000_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10000_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10000, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10002_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":0, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":1000, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10002_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10002, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Live nodes: [127.0.0.1:10000_solr] [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Collection testNodeLost state: DocCollection(testNodeLost//clusterstate.json/3)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "core":"testNodeLost_shard1_replica_n1", [junit4] 2> "leader":"true", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT"}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10001_solr", [junit4] 2> "core":"testNodeLost_shard1_replica_n2", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 126506 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 1 [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0 [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 2 [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10001_solr [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10002_solr [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [testNodeLost] [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 1 [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 1 [junit4] 2> 126507 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 1 [junit4] 2> 126508 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testNodeLost 1 [junit4] 2> 126508 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 1 [junit4] 2> 126508 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 126508 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1 [junit4] 2> 126508 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 3 [junit4] 2> 126509 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 126509 INFO (TEST-TestComputePlanAction.testNodeLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 126510 DEBUG (AutoscalingActionExecutor-493-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 446 ms for event id=8a51751d0a399Teyz8mtz9bcgiplhk5bgkx86jk [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestComputePlanAction -Dtests.method=testNodeLost -Dtests.seed=373B76FDD4175A98 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=asa -Dtests.timezone=Indian/Chagos -Dtests.asserts=true -Dtests.file.encoding=US-ASCII [junit4] ERROR 0.44s J0 | TestComputePlanAction.testNodeLost <<< [junit4] > Throwable #1: java.util.ConcurrentModificationException [junit4] > at __randomizedtesting.SeedInfo.seed([373B76FDD4175A98:882EB80357FD3F1E]:0) [junit4] > at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:939) [junit4] > at java.base/java.util.ArrayList$Itr.next(ArrayList.java:893) [junit4] > at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141) [junit4] > at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) [junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564) [junit4] > at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 126513 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded [junit4] 2> 126513 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 126513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 126513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 126513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126513 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr [junit4] 2> 126513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 126513 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10002_solr [junit4] 2> 126513 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 126513 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 126513 DEBUG (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1 [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 126514 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126534 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126555 DEBUG (ScheduledTrigger-495-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126575 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126595 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126614 DEBUG (simCloudManagerPool-494-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126614 DEBUG (simCloudManagerPool-494-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 126615 DEBUG (simCloudManagerPool-494-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 126615 DEBUG (simCloudManagerPool-494-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126615 DEBUG (simCloudManagerPool-494-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 126615 DEBUG (simCloudManagerPool-494-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9 [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9 [junit4] 2> 126616 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126636 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126656 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126676 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126696 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126716 DEBUG (simCloudManagerPool-494-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126716 DEBUG (simCloudManagerPool-494-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 126716 DEBUG (simCloudManagerPool-494-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126717 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126717 DEBUG (simCloudManagerPool-494-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126717 DEBUG (simCloudManagerPool-494-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 126717 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 126718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 126718 DEBUG (simCloudManagerPool-494-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126718 DEBUG (ScheduledTrigger-495-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 126718 DEBUG (ScheduledTrigger-495-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1 [junit4] 2> 126718 DEBUG (simCloudManagerPool-494-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 126718 DEBUG (simCloudManagerPool-494-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126718 DEBUG (simCloudManagerPool-494-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126718 DEBUG (simCloudManagerPool-494-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 126718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 126718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 126719 DEBUG (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeAdded [junit4] 2> 126729 WARN (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 2 on collection testNodeAdded is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance. [junit4] 2> 126729 DEBUG (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 126729 DEBUG (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 126730 INFO (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433338021 sessionWrapper.createTime 2433338008463961, this.sessionWrapper.createTime 2433338008463961 [junit4] 2> 126730 DEBUG (simCloudManagerPool-494-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 126737 DEBUG (ScheduledTrigger-495-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126739 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 126739 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1 [junit4] 2> 126752 DEBUG (simCloudManagerPool-494-thread-8) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126752 DEBUG (simCloudManagerPool-494-thread-8) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 126752 DEBUG (simCloudManagerPool-494-thread-8) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126752 DEBUG (simCloudManagerPool-494-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr] [junit4] 2> 126752 DEBUG (simCloudManagerPool-494-thread-8) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestComputePlanAction$AssertingTriggerAction}], enabled=true} [junit4] 2> 126752 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 126752 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 126757 DEBUG (ScheduledTrigger-495-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126759 DEBUG (ScheduledTrigger-495-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 126759 DEBUG (ScheduledTrigger-495-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 126759 DEBUG (ScheduledTrigger-495-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 2433339491750911 [junit4] 2> 126777 DEBUG (ScheduledTrigger-495-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 126779 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 126779 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 126779 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [2433339491750911], now=2433340506418761 [junit4] 2> 126780 DEBUG (ScheduledTrigger-495-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"8a51c000613ffTeyz8mtz9bcgiplhk5bgkx86jp", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2433339491750911, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433339491750911], [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 126780 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"8a51c000613ffTeyz8mtz9bcgiplhk5bgkx86jp", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2433339491750911, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433339491750911], [junit4] 2> "_enqueue_time_":2433340522108811, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 126780 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"8a51c000613ffTeyz8mtz9bcgiplhk5bgkx86jp", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":2433339491750911, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433339491750911], [junit4] 2> "_enqueue_time_":2433340522108811, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 126781 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 126781 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 126781 INFO (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeAdded&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node1 [junit4] 2> 126781 INFO (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433340600 sessionWrapper.createTime 2433340584946911, this.sessionWrapper.createTime 2433340584946911 [junit4] 2> 126781 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction Node values: { [junit4] 2> "127.0.0.1:10000_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":970, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10000_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10000, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10003_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":0, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":1000, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10003_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10003, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction Live nodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr], collection state: DocCollection(testNodeAdded//clusterstate.json/5)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 126782 DEBUG (AutoscalingActionExecutor-496-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 88 ms for event id=8a51c000613ffTeyz8mtz9bcgiplhk5bgkx86jp [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10000_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":970, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10000_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10000, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10003_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":0, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":1000, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10003_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10003, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}} [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Live nodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr] [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Collection testNodeAdded state: DocCollection(testNodeAdded//clusterstate.json/5)={ [junit4] 2> "replicationFactor":"2", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"2", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10000_solr", [junit4] 2> "core":"testNodeAdded_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active"}}} [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAdded [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 1 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [testNodeAdded] [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 2 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 2 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testNodeAdded 2 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 2 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 5 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 4 [junit4] 2> 126782 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 126783 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger [junit4] 2> 126783 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - AFTER_ACTION 1 [junit4] 2> 126783 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - BEFORE_ACTION 1 [junit4] 2> 126783 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 126783 INFO (TEST-TestComputePlanAction.testNodeAdded-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 126785 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeWithMultipleReplicasLost [junit4] 2> 126785 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 126785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 126785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 126785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 126786 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 126786 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 126788 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 126791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 126791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 126791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10003_solr] [junit4] 2> 126791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1 [junit4] 2> 126791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 126794 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126794 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 126794 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126814 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126834 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126854 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126875 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126891 DEBUG (simCloudManagerPool-497-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126891 DEBUG (simCloudManagerPool-497-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 126892 DEBUG (simCloudManagerPool-497-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10003_solr] [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15 [junit4] 2> 126892 DEBUG (simCloudManagerPool-497-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126892 DEBUG (simCloudManagerPool-497-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 126892 DEBUG (simCloudManagerPool-497-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10003_solr] [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126892 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 126895 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126915 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126935 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126955 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126975 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 [junit4] 2> 126993 DEBUG (simCloudManagerPool-497-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 126994 DEBUG (simCloudManagerPool-497-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 126994 DEBUG (simCloudManagerPool-497-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10003_solr] [junit4] 2> 126994 DEBUG (simCloudManagerPool-497-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10003_solr] [junit4] 2> 126994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 126994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 126994 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 126994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17 [junit4] 2> 126994 DEBUG (simCloudManagerPool-497-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeWithMultipleReplicasLost [junit4] 2> 126996 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 127005 DEBUG (simCloudManagerPool-497-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 127005 DEBUG (simCloudManagerPool-497-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 127007 INFO (simCloudManagerPool-497-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433351887 sessionWrapper.createTime 2433351776789211, this.sessionWrapper.createTime 2433351776789211 [junit4] 2> 127007 DEBUG (simCloudManagerPool-497-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 127014 DEBUG (ScheduledTrigger-498-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 127016 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 127035 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 127036 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 127055 DEBUG (ScheduledTrigger-498-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 4 [junit4] 2> 127056 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 [junit4] 2> 127070 DEBUG (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction -- cluster state: znodeVersion: 9 [junit4] 2> live nodes:[127.0.0.1:10004_solr, 127.0.0.1:10006_solr, 127.0.0.1:10003_solr, 127.0.0.1:10005_solr] [junit4] 2> collections:{testNodeWithMultipleReplicasLost=DocCollection(testNodeWithMultipleReplicasLost//clusterstate.json/8)={ [junit4] 2> "replicationFactor":"3", [junit4] 2> "pullReplicas":"0", [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"1", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"3", [junit4] 2> "tlogReplicas":"0", [junit4] 2> "autoCreated":"true", [junit4] 2> "shards":{ [junit4] 2> "shard2":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node6":{ [junit4] 2> "node_name":"127.0.0.1:10005_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node4":{ [junit4] 2> "node_name":"127.0.0.1:10004_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n4", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node5":{ [junit4] 2> "node_name":"127.0.0.1:10006_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n5", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"0-7fffffff", [junit4] 2> "state":"active"}, [junit4] 2> "shard1":{ [junit4] 2> "replicas":{ [junit4] 2> "core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10003_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node2":{ [junit4] 2> "node_name":"127.0.0.1:10005_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n2", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}, [junit4] 2> "core_node3":{ [junit4] 2> "node_name":"127.0.0.1:10006_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n3", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000}}, [junit4] 2> "range":"80000000-ffffffff", [junit4] 2> "state":"active"}}}} [junit4] 2> 127076 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 127076 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr [junit4] 2> 127076 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 127076 DEBUG (ScheduledTrigger-498-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr [junit4] 2> 127077 DEBUG (simCloudManagerPool-497-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard2: {"core_node6":{ [junit4] 2> "node_name":"127.0.0.1:10005_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard2_replica_n6", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard2", [junit4] 2> "collection":"testNodeWithMultipleReplicasLost"}} [junit4] 2> 127077 DEBUG (simCloudManagerPool-497-thread-12) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeWithMultipleReplicasLost / shard1: {"core_node1":{ [junit4] 2> "node_name":"127.0.0.1:10003_solr", [junit4] 2> "core":"testNodeWithMultipleReplicasLost_shard1_replica_n1", [junit4] 2> "leader":"true", [junit4] 2> "state":"active", [junit4] 2> "type":"NRT", [junit4] 2> "INDEX.sizeInBytes":123450000, [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"testNodeWithMultipleReplicasLost"}} [junit4] 2> 127097 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 127097 DEBUG (ScheduledTrigger-498-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 127097 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10006_solr] [junit4] 2> 127097 DEBUG (ScheduledTrigger-498-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"8a51fb1b01c0bTeyz8mtz9bcgiplhk5bgkx86ju", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433355357363211, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433355357363211], [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 127097 DEBUG (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"8a51fb1b01c0bTeyz8mtz9bcgiplhk5bgkx86ju", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433355357363211, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433355357363211], [junit4] 2> "_enqueue_time_":2433356376403061, [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} [junit4] 2> 127097 DEBUG (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: { [junit4] 2> "id":"8a51fb1b01c0bTeyz8mtz9bcgiplhk5bgkx86ju", [junit4] 2> "source":"node_lost_trigger", [junit4] 2> "eventTime":2433355357363211, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[2433355357363211], [junit4] 2> "_enqueue_time_":2433356376403061, [junit4] 2> "nodeNames":["127.0.0.1:10006_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]} [junit4] 2> 127097 DEBUG (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session [junit4] 2> 127098 DEBUG (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created [junit4] 2> 127099 INFO (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5 [junit4] 2> 127099 INFO (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node3 [junit4] 2> 127099 INFO (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2433356498 sessionWrapper.createTime 2433356434370861, this.sessionWrapper.createTime 2433356434370861 [junit4] 2> 127099 DEBUG (AutoscalingActionExecutor-499-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL [junit4] 2> 127100 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node5 [junit4] 2> 127100 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction action=MOVEREPLICA&collection=testNodeWithMultipleReplicasLost&targetNode=127.0.0.1:10004_solr&inPlaceMove=true&replica=core_node3 [junit4] 2> 127100 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction -------------_ FINAL STATE -------------- [junit4] 2> 127100 INFO (TEST-TestComputePlanAction.testNodeWithMultipleReplicasLost-seed#[373B76FDD4175A98]) [ ] o.a.s.c.a.s.TestComputePlanAction * Node values: { [junit4] 2> "127.0.0.1:10004_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":1, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":990, [junit4] 2> "sysprop.java.version":"9.0.1", [junit4] 2> "ip_1":"127", [junit4] 2> "ip_3":"0", [junit4] 2> "ip_2":"0", [junit4] 2> "ip_4":"1", [junit4] 2> "node":"127.0.0.1:10004_solr", [junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0, [junit4] 2> "port":10004, [junit4] 2> "heapUsage":123450000, [junit4] 2> "sysLoadAvg":1.0, [junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}, [junit4] 2> "127.0.0.1:10006_solr":{ [junit4] 2> "sysprop.java.vendor":"Oracle Corporation", [junit4] 2> "cores":2, [junit4] 2> "host":"127.0.0.1", [junit4] 2> "freedisk":980, [junit4] 2> "sysprop.java.version":"9.0.1", [junit [...truncated too long message...] 4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:1014) [junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 1376341 ERROR (coreZkRegister-5206-thread-1-processing-n:127.0.0.1:37235_solr x:awhollynewcollection_0_shard2_replica_n6 s:shard2 c:awhollynewcollection_0 r:core_node9) [n:127.0.0.1:37235_solr c:awhollynewcollection_0 s:shard2 r:core_node9 x:awhollynewcollection_0_shard2_replica_n6] o.a.s.c.ZkContainer [junit4] 2> java.lang.NullPointerException [junit4] 2> at org.apache.solr.cloud.ZkController.publish(ZkController.java:1350) [junit4] 2> at org.apache.solr.cloud.ZkController.publish(ZkController.java:1342) [junit4] 2> at org.apache.solr.cloud.ZkController.publish(ZkController.java:1338) [junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:188) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> 1376341 ERROR (coreZkRegister-5206-thread-1-processing-n:127.0.0.1:37235_solr x:awhollynewcollection_0_shard2_replica_n6 s:shard2 c:awhollynewcollection_0 r:core_node9) [n:127.0.0.1:37235_solr c:awhollynewcollection_0 s:shard2 r:core_node9 x:awhollynewcollection_0_shard2_replica_n6] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard2 [junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1207) [junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:1079) [junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:1014) [junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) [junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) [junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844) [junit4] 2> Caused by: org.apache.solr.common.SolrException: CoreContainer is closed [junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1251) [junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1218) [junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1174) [junit4] 2> ... 7 more [junit4] 2> [junit4] 2> 1376341 INFO (jetty-closer-2607-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4775dfe2{/solr,null,UNAVAILABLE} [junit4] 2> 1377773 INFO (jetty-closer-2607-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@65b16ec1{/solr,null,UNAVAILABLE} [junit4] 2> 1377773 ERROR (SUITE-CollectionsAPIDistributedZkTest-seed#[373B76FDD4175A98]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 1377774 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[373B76FDD4175A98]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41415 41415 [junit4] 2> 1377780 INFO (Thread-2462) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41415 41415 [junit4] 2> 1377781 WARN (Thread-2462) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 22 /solr/configs/conf [junit4] 2> 18 /solr/aliases.json [junit4] 2> 18 /solr/clusterprops.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 36 /solr/collections/addReplicaColl/state.json [junit4] 2> 32 /solr/collections/nodes_used_collection/state.json [junit4] 2> 31 /solr/collections/reloaded_collection/state.json [junit4] 2> 18 /solr/clusterstate.json [junit4] 2> 13 /solr/collections/acollectionafterbaddelete/state.json [junit4] 2> 13 /solr/collections/halfdeletedcollection/state.json [junit4] 2> 10 /solr/collections/onlyinzk/state.json [junit4] 2> 5 /solr/collections/created_and_deleted/state.json [junit4] 2> 5 /solr/collections/withconfigset2/state.json [junit4] 2> 4 /solr/collections/halfcollectionblocker/state.json [junit4] 2> 4 /solr/collections/halfcollection/state.json [junit4] 2> 4 /solr/collections/halfcollectionblocker2/state.json [junit4] 2> 2 /solr/collections/awhollynewcollection_0/state.json [junit4] 2> 2 /solr/collections/addReplicaColl/leader_elect/shard1/election/99269562558316554-core_node3-n_0000000000 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 302 /solr/collections [junit4] 2> 18 /solr/live_nodes [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_373B76FDD4175A98-001 [junit4] 2> Dec 31, 2017 3:16:40 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Direct)}, docValues:{}, maxPointsInLeafNode=47, maxMBSortInHeap=5.143104228115581, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@61892a39), locale=gd, timezone=Indian/Maldives [junit4] 2> NOTE: Linux 4.10.0-40-generic amd64/Oracle Corporation 9.0.1 (64-bit)/cpus=8,threads=1,free=81457552,total=523763712 [junit4] 2> NOTE: All tests run in this JVM: [TestRTGBase, SuggesterTSTTest, TestLocalFSCloudBackupRestore, TestNumericTokenStream, TestSurroundQueryParser, TestManagedSynonymFilterFactory, TestTrackingShardHandlerFactory, TestAnalyzedSuggestions, TestEmbeddedSolrServerAdminHandler, HighlighterConfigTest, SuggesterFSTTest, TestDynamicFieldResource, TestSimpleTrackingShardHandler, BlockJoinFacetDistribTest, NumericFieldsTest, DirectoryFactoryTest, TestMultiValuedNumericRangeQuery, HdfsChaosMonkeyNothingIsSafeTest, TestSolrQueryParser, LukeRequestHandlerTest, DirectUpdateHandlerTest, TestXIncludeConfig, CdcrReplicationDistributedZkTest, TestPointFields, AssignTest, TestInitQParser, TestDocumentBuilder, ConcurrentDeleteAndCreateCollectionTest, TestCorePropertiesReload, SolrMetricReporterTest, TestUseDocValuesAsStored, DistributedFacetPivotWhiteBoxTest, BaseCdcrDistributedZkTest, ShardRoutingCustomTest, TestAuthorizationFramework, OutputWriterTest, TestSolrCloudWithHadoopAuthPlugin, OpenCloseCoreStressTest, TestUtilizeNode, TestCloudSchemaless, ChaosMonkeySafeLeaderWithPullReplicasTest, MetricTriggerTest, TestMacros, TestClusterProperties, HdfsNNFailoverTest, TestBM25SimilarityFactory, TestJettySolrRunner, AddBlockUpdateTest, TestShortCircuitedRequests, TestOmitPositions, TestLargeCluster, TestSolr4Spatial2, TestSolr4Spatial, TestOverriddenPrefixQueryForCustomFieldType, TestStressReorder, ExplicitHLLTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsLockFactoryTest, SearchHandlerTest, TestLegacyFieldReuse, TermVectorComponentDistributedTest, AnalyticsQueryTest, ManagedSchemaRoundRobinCloudTest, TestRangeQuery, TestRequestStatusCollectionAPI, DistributedSuggestComponentTest, DateRangeFieldTest, BinaryUpdateRequestHandlerTest, NotRequiredUniqueKeyTest, ZkCLITest, TestCloudInspectUtil, DeleteLastCustomShardedReplicaTest, BasicAuthIntegrationTest, PropertiesRequestHandlerTest, ShowFileRequestHandlerTest, SynonymTokenizerTest, TestDistribStateManager, TestPolicyCloud, MoreLikeThisHandlerTest, RulesTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DocumentAnalysisRequestHandlerTest, SpatialRPTFieldTypeTest, TestFreeTextSuggestions, TestCryptoKeys, BadIndexSchemaTest, TestPseudoReturnFields, TestAnalyzeInfixSuggestions, TestInPlaceUpdatesDistrib, SchemaVersionSpecificBehaviorTest, PrimUtilsTest, OverseerTaskQueueTest, SpellingQueryConverterTest, TestPullReplicaErrorHandling, TestConfigSetsAPI, TestDownShardTolerantSearch, TestManagedSchema, TestSolrCloudWithDelegationTokens, BooleanFieldTest, TestSubQueryTransformer, VMParamsZkACLAndCredentialsProvidersTest, TestDocSet, ZkStateWriterTest, SOLR749Test, TestExpandComponent, ShardRoutingTest, RequestHandlersTest, SuggestComponentContextFilterQueryTest, AddReplicaTest, DOMUtilTest, TestComplexPhraseQParserPlugin, SolrXmlInZkTest, TestFieldTypeCollectionResource, TestQuerySenderNoQuery, SolrRequestParserTest, TestLazyCores, DistributedIntervalFacetingTest, LeaderFailoverAfterPartitionTest, TestSolrCloudSnapshots, SolrSlf4jReporterTest, ResponseHeaderTest, TestDFISimilarityFactory, ClassificationUpdateProcessorTest, TestSQLHandlerNonCloud, QueryResultKeyTest, BasicDistributedZkTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest] [junit4] Completed [381/765 (3!)] on J1 in 85.75s, 19 tests, 1 error <<< FAILURES! [...truncated 42021 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
