Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4208/ Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
4 tests failed. FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI Error Message: Error from server at https://127.0.0.1:64458/solr/awhollynewcollection_0: {"awhollynewcollection_0":5} Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:64458/solr/awhollynewcollection_0: {"awhollynewcollection_0":5} at __randomizedtesting.SeedInfo.seed([FD147F641C331622:B5610BD01A0039B7]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:967) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest Error Message: Could not find collection : delLiveColl Stack Trace: org.apache.solr.common.SolrException: Could not find collection : delLiveColl at __randomizedtesting.SeedInfo.seed([FD147F641C331622:5074CB6F010CBE57]:0) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:111) at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247) at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:51) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaByCountForAllShards Error Message: Expected two shards with two replicas each null Live Nodes: [127.0.0.1:52370_solr, 127.0.0.1:52371_solr, 127.0.0.1:52372_solr, 127.0.0.1:52373_solr] Last available state: null Stack Trace: java.lang.AssertionError: Expected two shards with two replicas each null Live Nodes: [127.0.0.1:52370_solr, 127.0.0.1:52371_solr, 127.0.0.1:52372_solr, 127.0.0.1:52373_solr] Last available state: null at __randomizedtesting.SeedInfo.seed([FD147F641C331622:C185383053724B77]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaByCountForAllShards(DeleteReplicaTest.java:137) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry Error Message: Captured an uncaught exception in thread: Thread[id=11635, name=OverseerAutoScalingTriggerThread-98774722066317314-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers] Stack Trace: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=11635, name=OverseerAutoScalingTriggerThread-98774722066317314-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers] at __randomizedtesting.SeedInfo.seed([FD147F641C331622:2BCB5E3E2A2E05BA]:0) Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas at __randomizedtesting.SeedInfo.seed([FD147F641C331622]:0) at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783) at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:527) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:524) at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:97) at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105) ... 5 more Build Log: [...truncated 12223 lines...] [junit4] Suite: org.apache.solr.cloud.TestLeaderElectionZkExpiry [junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestLeaderElectionZkExpiry_FD147F641C331622-001/init-core-data-001 [junit4] 2> 1224721 WARN (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14 [junit4] 2> 1224722 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 1224726 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth [junit4] 2> 1224726 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 1224727 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLeaderElectionWithZkExpiry [junit4] 2> 1224733 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@56e1771d, but no JMX reporters were configured - adding default JMX reporter. [junit4] 2> 1224775 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@56e1771d [junit4] 2> 1224781 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@56e1771d [junit4] 2> 1224781 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@56e1771d [junit4] 2> 1224782 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestLeaderElectionZkExpiry_FD147F641C331622-001/testLeaderElectionWithZkExpiry-solr-001 [junit4] 2> 1224783 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 1224783 INFO (Thread-2567) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 1224783 INFO (Thread-2567) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 1224785 ERROR (Thread-2567) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 1224884 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [ ] o.a.s.c.ZkTestServer start zk server on port:64374 [junit4] 2> 1225048 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 1225049 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr [junit4] 2> 1225050 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=98774722066317314-dummy.host.com:8984_solr-n_0000000000) starting [junit4] 2> 1225062 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr [junit4] 2> 1225064 INFO (zkCallback-2072-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 1225065 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x15eeb161d8d0002 due to java.nio.channels.CancelledKeyException [junit4] 2> 1225174 WARN (zkCallback-2072-thread-4-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1225174 WARN (zkCallback-2072-thread-4-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1226549 WARN (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0002 has expired [junit4] 2> 1226549 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1226549 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1226549 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=98774722066317314-dummy.host.com:8984_solr-n_0000000000) closing [junit4] 2> 1226549 WARN (OverseerCollectionConfigSetProcessor-98774722066317314-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor Overseer cannot talk to ZK [junit4] 2> 1226549 INFO (OverseerStateUpdate-98774722066317314-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr [junit4] 2> 1226561 ERROR (OverseerExitThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer could not read the data [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:299) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1226674 WARN (zkCallback-2072-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> lis 05, 2017 2:50:50 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException [junit4] 2> WARNING: Uncaught exception in thread: Thread[OverseerAutoScalingTriggerThread-98774722066317314-dummy.host.com:8984_solr-n_0000000000,5,Overseer autoscaling triggers] [junit4] 2> org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas [junit4] 2> at __randomizedtesting.SeedInfo.seed([FD147F641C331622]:0) [junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107) [junit4] 2> at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:527) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:524) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:97) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) [junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1226690 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1226691 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1226691 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1226692 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 1226698 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception [junit4] 2> java.nio.channels.CancelledKeyException [junit4] 2> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) [junit4] 2> at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1226798 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1226798 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1228251 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0003 has expired [junit4] 2> 1228251 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1228251 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1228251 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1228254 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1228254 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1228261 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Session 0x15eeb161d8d0004 for server 127.0.0.1/127.0.0.1:64374, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Connection reset by peer [junit4] 2> at sun.nio.ch.FileDispatcherImpl.read0(Native Method) [junit4] 2> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) [junit4] 2> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) [junit4] 2> at sun.nio.ch.IOUtil.read(IOUtil.java:192) [junit4] 2> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:68) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) [junit4] 2> 1228372 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1228372 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1229971 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0004 has expired [junit4] 2> 1229971 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1229971 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1229972 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1229974 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1229974 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1230075 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /security.json [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:306) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:303) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.addSecuritynodeWatcher(ZkStateReader.java:491) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:478) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> [junit4] 2> 1230076 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /security.json [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:306) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:303) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.addSecuritynodeWatcher(ZkStateReader.java:491) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:478) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1230076 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1230077 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1230090 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1230090 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1231516 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0005 has expired [junit4] 2> 1231516 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1231517 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1231517 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1231519 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1231519 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1231621 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:105) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> [junit4] 2> 1231622 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:105) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> ... 10 more [junit4] 2> [junit4] 2> 1231622 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1231622 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1231626 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1231626 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1232767 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0006 has expired [junit4] 2> 1232767 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1232768 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1232768 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1232769 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1232769 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1232883 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1232883 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1233993 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0007 has expired [junit4] 2> 1233993 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1233994 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1233994 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1233996 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1233996 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1234098 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None [junit4] 2> 1234098 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected [junit4] 2> 1235202 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64374)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15eeb161d8d0008 has expired [junit4] 2> 1235202 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62f8d058 name: ZooKeeperConnection Watcher:127.0.0.1:64374/solr got event WatchedEvent state:Expired type:None path:null path: null type: None [junit4] 2> 1235202 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper... [junit4] 2> 1235202 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one... [junit4] 2> 1235204 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished. [junit4] 2> 1235204 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration. [junit4] 2> 1235303 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> [junit4] 2> 1235303 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1235303 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1235304 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1235312 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr [junit4] 2> 1235314 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=98774722066317322-dummy.host.com:8984_solr-n_0000000002) starting [junit4] 2> 1235315 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.TestLeaderElectionZkExpiry Time=1096172741738872 Overseer leader is = dummy.host.com:8984_solr [junit4] 2> 1235325 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr [junit4] 2> 1235325 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=98774722066317322-dummy.host.com:8984_solr-n_0000000002) closing [junit4] 2> 1235325 WARN (OverseerAutoScalingTriggerThread-98774722066317322-dummy.host.com:8984_solr-n_0000000002) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.lang.Object.wait(Native Method) [junit4] 2> at java.lang.Object.wait(Object.java:502) [junit4] 2> at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:185) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getAutoScalingConfig(ZkStateReader.java:171) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:116) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1235326 INFO (OverseerCollectionConfigSetProcessor-98774722066317322-dummy.host.com:8984_solr-n_0000000002) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98774722066317322-dummy.host.com:8984_solr-n_0000000002) am no longer a leader. [junit4] 2> 1235326 INFO (OverseerStateUpdate-98774722066317322-dummy.host.com:8984_solr-n_0000000002) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer According to ZK I (id=98774722066317322-dummy.host.com:8984_solr-n_0000000002) am no longer a leader. [junit4] 2> 1235326 INFO (OverseerStateUpdate-98774722066317322-dummy.host.com:8984_solr-n_0000000002) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr [junit4] 2> 1235326 ERROR (OverseerCollectionConfigSetProcessor-98774722066317322-dummy.host.com:8984_solr-n_0000000002) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor Unable to prioritize overseer [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.lang.Object.wait(Native Method) [junit4] 2> at java.lang.Object.wait(Object.java:502) [junit4] 2> at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1097) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.cloud.OverseerNodePrioritizer.prioritizeOverseerNodes(OverseerNodePrioritizer.java:61) [junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:178) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 1235329 INFO (zkCallback-2072-thread-6-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 1235329 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper [junit4] 2> 1235331 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1235328 WARN (OverseerExitThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader [junit4] 2> 1235333 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> [junit4] 2> 1235333 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> [junit4] 2> 1235333 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1235333 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1235333 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267) [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1235334 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1235334 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1235335 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1236832 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) [junit4] 2> at org.apache.solr.cloud.LeaderElector.setup(LeaderElector.java:364) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:346) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> [junit4] 2> 1236832 ERROR (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168) [junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) [junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) [junit4] 2> at org.apache.solr.cloud.LeaderElector.setup(LeaderElector.java:364) [junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:346) [junit4] 2> ... 5 more [junit4] 2> [junit4] 2> 1236832 WARN (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed [junit4] 2> 1236833 INFO (zkCallback-2072-thread-2-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connected:true [junit4] 2> 1236833 WARN (zkCallback-2072-thread-7-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 1236833 WARN (OverseerExitThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Unable to rejoinElection [junit4] 2> org.apache.solr.common.SolrException: Unable to rejoin election [junit4] 2> at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:1924) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:324) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect [junit4] 2> at org.apache.solr.cloud.OverseerElectionContext.<init>(ElectionContext.java:772) [junit4] 2> at org.apache.solr.cloud.OverseerElectionContext.copy(ElectionContext.java:815) [junit4] 2> at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:385) [junit4] 2> at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:1921) [junit4] 2> ... 2 more [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) [junit4] 2> at org.apache.solr.cloud.OverseerElectionContext.<init>(ElectionContext.java:770) [junit4] 2> ... 5 more [junit4] 2> 1236833 WARN (zkCallback-2072-thread-6-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 1236833 WARN (zkCallback-2072-thread-4-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 1236833 WARN (zkCallback-2072-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 1236835 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=185838429 [junit4] 2> 1236835 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null [junit4] 2> 1236835 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31cf3943: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@73ef7091 [junit4] 2> 1236839 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null [junit4] 2> 1236839 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@68025e62: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@63d91bc8 [junit4] 2> 1236841 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null [junit4] 2> 1236842 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@24df763e: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@18698592 [junit4] 2> 1236842 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 1236842 ERROR (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 1236843 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64374 64374 [junit4] 2> 1236869 INFO (Thread-2567) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64374 64374 [junit4] 2> 1236997 WARN (Thread-2567) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 7 /solr/aliases.json [junit4] 2> 7 /solr/clusterprops.json [junit4] 2> 6 /solr/security.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 7 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 9 /solr/live_nodes [junit4] 2> 7 /solr/collections [junit4] 2> [junit4] 2> 1236998 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[FD147F641C331622]) [n:dummy.host.com:8984_solr ] o.a.s.SolrTestCaseJ4 ###Ending testLeaderElectionWithZkExpiry [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestLeaderElectionZkExpiry -Dtests.method=testLeaderElectionWithZkExpiry -Dtests.seed=FD147F641C331622 -Dtests.slow=true -Dtests.locale=hr-HR -Dtests.timezone=America/Bahia -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1 [junit4] ERROR 12.3s J1 | TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry <<< [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=11635, name=OverseerAutoScalingTriggerThread-98774722066317314-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers] [junit4] > at __randomizedtesting.SeedInfo.seed([FD147F641C331622:2BCB5E3E2A2E05BA]:0) [junit4] > Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas [junit4] > at __randomizedtesting.SeedInfo.seed([FD147F641C331622]:0) [junit4] > at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107) [junit4] > at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44) [junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398) [junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149) [junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas [junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] > at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783) [junit4] > at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:527) [junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:524) [junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:97) [junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) [junit4] > at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105) [junit4] > ... 5 more [junit4] 2> 1237021 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> 1237021 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=185838429 [junit4] 2> 1237021 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null [junit4] 2> 1237021 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null [junit4] 2> 1237021 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null [junit4] 2> 1237022 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[FD147F641C331622]-worker) [n:dummy.host.com:8984_solr ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released [junit4] 2> Oct 05, 2017 5:51:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1459, maxMBSortInHeap=7.7511081743531705, sim=RandomSimilarity(queryNorm=true): {}, locale=hr-HR, timezone=America/Bahia [junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=191288232,total=441450496 [junit4] 2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, TestCollationField, TestSolrDeletionPolicy1, CoreMergeIndexesAdminHandlerTest, TestDistributedMissingSort, TestEmbeddedSolrServerConstructors, SpellCheckComponentTest, TestLocalFSCloudBackupRestore, JvmMetricsTest, TriggerIntegrationTest, ZkStateWriterTest, ConfigSetsAPITest, CacheHeaderTest, AnalysisErrorHandlingTest, TestFastWriter, TestSegmentSorting, TestJettySolrRunner, TestFastLRUCache, V2StandaloneTest, DOMUtilTest, SolrSlf4jReporterTest, TestReplicaProperties, SpellCheckCollatorTest, TestSimpleQParserPlugin, TestNumericTerms32, TestCoreDiscovery, SolrCoreCheckLockOnStartupTest, TestDocSet, BlockJoinFacetSimpleTest, ShardRoutingCustomTest, TestFieldTypeResource, PluginInfoTest, TestConfigSetsAPIZkFailure, ActionThrottleTest, CursorMarkTest, DistanceUnitsTest, SolrXmlInZkTest, LeaderFailoverAfterPartitionTest, TestRangeQuery, TestPostingsSolrHighlighter, SolrGangliaReporterTest, MergeStrategyTest, HdfsAutoAddReplicasIntegrationTest, TestMinMaxOnMultiValuedField, TestElisionMultitermQuery, SynonymTokenizerTest, TestInfoStreamLogging, TestAnalyzedSuggestions, TestSchemaManager, SignatureUpdateProcessorFactoryTest, LeaderElectionIntegrationTest, TestDefaultStatsCache, TestUnifiedSolrHighlighter, TestManagedSchemaThreadSafety, TestDelegationWithHadoopAuth, TestSuggestSpellingConverter, TestSolrCLIRunExample, TestSolrIndexConfig, CopyFieldTest, TestJoin, TestComponentsName, SearchHandlerTest, NodeMutatorTest, TestLuceneMatchVersion, TestPayloadScoreQParserPlugin, CheckHdfsIndexTest, TestJsonRequest, ZkFailoverTest, TestSolrCloudSnapshots, PreAnalyzedFieldManagedSchemaCloudTest, TestNumericRangeQuery32, TestFiltering, DebugComponentTest, BasicAuthStandaloneTest, TestExpandComponent, TestBadConfig, StatsComponentTest, TestTlogReplica, IgnoreCommitOptimizeUpdateProcessorFactoryTest, RuleEngineTest, TestTrie, DistributedFacetPivotLargeTest, TriLevelCompositeIdRoutingTest, SolrCoreMetricManagerTest, TestMergePolicyConfig, HLLUtilTest, RollingRestartTest, BasicFunctionalityTest, SaslZkACLProviderTest, TestValueSourceCache, ReplicationFactorTest, LoggingHandlerTest, DocumentAnalysisRequestHandlerTest, TestDynamicFieldResource, SolrJmxReporterCloudTest, SuggesterFSTTest, TestClusterStateMutator, PolyFieldTest, CollectionReloadTest, TestSchemaResource, ChaosMonkeySafeLeaderWithPullReplicasTest, SOLR749Test, TestFileDictionaryLookup, BlockJoinFacetDistribTest, TestRandomRequestDistribution, TestRandomFaceting, PingRequestHandlerTest, AutoAddReplicasPlanActionTest, TestHdfsBackupRestoreCore, BasicZkTest, TestLegacyFieldReuse, StatsReloadRaceTest, SpellingQueryConverterTest, TestCloudPseudoReturnFields, BadComponentTest, TestSolrFieldCacheBean, QueryEqualityTest, TestSystemCollAutoCreate, TestRestManager, TestFieldResource, TestConfigSetsAPI, TestHdfsCloudBackupRestore, MBeansHandlerTest, OverseerStatusTest, SortSpecParsingTest, DeleteShardTest, DistributedQueryElevationComponentTest, ForceLeaderTest, TestSizeLimitedDistributedMap, TestUninvertingReader, MoveReplicaHDFSFailoverTest, ConvertedLegacyTest, TestPerFieldSimilarityWithDefaultOverride, TestSubQueryTransformerCrossCore, UtilsToolTest, TestBlobHandler, TestReload, SpatialHeatmapFacetsTest, DeleteNodeTest, SolrCmdDistributorTest, TestNRTOpen, TestLeaderElectionZkExpiry] [junit4] Completed [245/745 (1!)] on J1 in 12.57s, 1 test, 1 error <<< FAILURES! [...truncated 47 lines...] [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest [junit4] 2> 1238085 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FD147F641C331622]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FD147F641C331622-001/init-core-data-001 [junit4] 2> 1238085 WARN (SUITE- [...truncated too long message...] n = solr.core.deleteByCount.shard1.replica_t1, service url = null, agent id = null] for registry solr.core.deleteByCount.shard1.replica_t1 / com.codahale.metrics.MetricRegistry@741e07ea [junit4] 2> 2292460 INFO (coreCloseExecutor-9726-thread-2) [n:127.0.0.1:52370_solr c:deleteByCount s:shard1 r:core_node3 x:deleteByCount_shard1_replica_t1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteByCount.shard1.leader, tag=1581571182 [junit4] 2> 2292462 WARN (zkCallback-4697-thread-4-processing-n:127.0.0.1:52370_solr) [n:127.0.0.1:52370_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 2292462 INFO (jetty-closer-4676-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@584044d4{/solr,null,UNAVAILABLE} [junit4] 2> 2293947 WARN (zkCallback-4694-thread-4-processing-n:127.0.0.1:52372_solr) [n:127.0.0.1:52372_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 2293948 INFO (jetty-closer-4676-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3862da7b{/solr,null,UNAVAILABLE} [junit4] 2> 2293949 ERROR (SUITE-DeleteReplicaTest-seed#[FD147F641C331622]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 2293949 INFO (SUITE-DeleteReplicaTest-seed#[FD147F641C331622]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52368 52368 [junit4] 2> 2293963 INFO (Thread-4317) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52368 52368 [junit4] 2> 2294134 WARN (Thread-4317) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 5 /solr/clusterprops.json [junit4] 2> 5 /solr/aliases.json [junit4] 2> 4 /solr/security.json [junit4] 2> 4 /solr/collections/delLiveColl/state.json [junit4] 2> 4 /solr/configs/conf [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 23 /solr/collections/deleteByCount/state.json [junit4] 2> 15 /solr/collections/deletereplica_test/state.json [junit4] 2> 12 /solr/collections/delLiveColl/state.json [junit4] 2> 5 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 5 /solr/live_nodes [junit4] 2> 5 /solr/collections [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_FD147F641C331622-001 [junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=zh-SG, timezone=Africa/Johannesburg [junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=43890216,total=517996544 [junit4] 2> NOTE: All tests run in this JVM: [TestNumericRangeQuery64, TestSurroundQueryParser, DateFieldTest, TestFuzzyAnalyzedSuggestions, SimpleFacetsTest, PeerSyncTest, TestDistributedSearch, TestCloudInspectUtil, SparseHLLTest, TestRandomFlRTGCloud, TestStressLiveNodes, RulesTest, MissingSegmentRecoveryTest, TestNamedUpdateProcessors, TestFieldCacheSort, TestClusterProperties, BooleanFieldTest, BJQParserTest, TestEmbeddedSolrServerAdminHandler, TestStressRecovery, AlternateDirectoryTest, TestGroupingSearch, ZkCLITest, TestConfigSetsAPIExclusivity, TestLegacyFieldCache, TestSimpleTextCodec, TestTestInjection, RAMDirectoryFactoryTest, HdfsBasicDistributedZk2Test, TestExclusionRuleCollectionAccess, TestCloudPivotFacet, TestUtils, TestOnReconnectListenerSupport, TestImpersonationWithHadoopAuth, TestPKIAuthenticationPlugin, TestCryptoKeys, TestPullReplica, ResponseHeaderTest, MultiTermTest, TestJsonFacets, SolrCoreTest, TestOmitPositions, SuggesterTest, ResponseLogComponentTest, PKIAuthenticationIntegrationTest, DistributedFacetPivotLongTailTest, MultiThreadedOCPTest, TestSSLRandomization, TestMissingGroups, TestReplicationHandler, StressHdfsTest, CdcrReplicationDistributedZkTest, TestBackupRepositoryFactory, CloudExitableDirectoryReaderTest, TestCustomSort, TestFunctionQuery, RequestHandlersTest, TestApiFramework, ExplicitHLLTest, UUIDFieldTest, HighlighterMaxOffsetTest, HdfsCollectionsAPIDistributedZkTest, TestFieldCache, TestCSVResponseWriter, BufferStoreTest, TestSolrCoreParser, HardAutoCommitTest, HdfsChaosMonkeyNothingIsSafeTest, ParsingFieldUpdateProcessorsTest, SolrMetricsIntegrationTest, TestDocBasedVersionConstraints, TestSlowCompositeReaderWrapper, PropertiesRequestHandlerTest, InfoHandlerTest, TestReloadAndDeleteDocs, TestBulkSchemaAPI, LeaderFailureAfterFreshStartTest, ExecutePlanActionTest, TestXmlQParserPlugin, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DateMathParserTest, CloudMLTQParserTest, TestSolrConfigHandlerConcurrent, TestSerializedLuceneMatchVersion, DocValuesNotIndexedTest, LeaderInitiatedRecoveryOnShardRestartTest, TestSolrQueryResponse, DocExpirationUpdateProcessorFactoryTest, TestPseudoReturnFields, TestClassicSimilarityFactory, SSLMigrationTest, TestSchemaSimilarityResource, CSVRequestHandlerTest, SegmentsInfoRequestHandlerTest, ShardSplitTest, TestDistributedStatsComponentCardinality, TestSubQueryTransformer, BasicDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestFaceting, BlockCacheTest, TestSchemaNameResource, SimplePostToolTest, TestLazyCores, TestIndexingPerformance, TestExportWriter, TestConfigReload, OutOfBoxZkACLAndCredentialsProvidersTest, DeleteStatusTest, TestOrdValues, NodeLostTriggerTest, UniqFieldsUpdateProcessorFactoryTest, DirectSolrConnectionTest, TestBinaryResponseWriter, DistributedSuggestComponentTest, TestInitParams, TestCloudSchemaless, RankQueryTest, SpellCheckCollatorWithCollapseTest, AnalyticsQueryTest, BlockJoinFacetRandomTest, TestSQLHandlerNonCloud, LargeFieldTest, TestCoreContainer, VMParamsZkACLAndCredentialsProvidersTest, ManagedSchemaRoundRobinCloudTest, SearchHandlerTest, ImplicitSnitchTest, ShardRoutingTest, TestTrackingShardHandlerFactory, TestStressCloudBlindAtomicUpdates, WrapperMergePolicyFactoryTest, CoreAdminRequestStatusTest, DirectUpdateHandlerOptimizeTest, TestPayloadCheckQParserPlugin, SpatialFilterTest, XsltUpdateRequestHandlerTest, MinimalSchemaTest, SubstringBytesRefFilterTest, CollectionStateFormat2Test, NumericFieldsTest, TestStandardQParsers, TestManagedResourceStorage, DistributedSpellCheckComponentTest, HighlighterTest, TestQuerySenderListener, OverseerCollectionConfigSetProcessorTest, BigEndianAscendingWordSerializerTest, TestNoOpRegenerator, ConfigureRecoveryStrategyTest, SolrCloudExampleTest, TestRandomDVFaceting, TestStressVersions, TestTolerantUpdateProcessorCloud, TestAtomicUpdateErrorCases, SolrTestCaseJ4Test, AutoScalingHandlerTest, DistributedFacetExistsSmallTest, TestCodecSupport, SoftAutoCommitTest, TestDistribIDF, CurrencyFieldTypeTest, PathHierarchyTokenizerFactoryTest, CursorPagingTest, DeleteLastCustomShardedReplicaTest, TestFieldCollectionResource, TestExactStatsCache, TestConfigSetImmutable, TestPushWriter, QueryElevationComponentTest, TestZkAclsWithHadoopAuth, PrimitiveFieldTypeTest, TestUpdate, TestLeaderElectionWithEmptyReplica, TestRestoreCore, TestMultiValuedNumericRangeQuery, RecoveryZkTest, TestSkipOverseerOperations, DocValuesTest, LeaderElectionContextKeyTest, TestPHPSerializedResponseWriter, OverseerRolesTest, OpenCloseCoreStressTest, OverseerTest, ClusterStateUpdateTest, TestZkChroot, TestDistributedGrouping, TestRecovery, TestHashPartitioner, ZkControllerTest, TestRealTimeGet, DirectUpdateHandlerTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTSTTest, TestStressLucene, TestCSVLoader, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, FileBasedSpellCheckerTest, TermVectorComponentTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, RegexBoostProcessorTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, JsonLoaderTest, TestLFUCache, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TimeZoneUtilsTest, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PreAnalyzedFieldTest, ClusterStateTest, TestDocumentBuilder, CircularListTest, TestRTGBase, DistributedIntervalFacetingTest, AliasIntegrationTest, AssignTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteReplicaTest] [junit4] Completed [518/745 (3!)] on J0 in 167.91s, 4 tests, 1 failure, 1 error <<< FAILURES! [...truncated 47607 lines...] BUILD FAILED /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:826: The following error occurred while executing this line: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:706: The following error occurred while executing this line: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:693: Source checkout is dirty (unversioned/missing files) after running tests!!! Offending files: * lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1 Total time: 91 minutes 7 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts [WARNINGS] Skipping publisher since build result is FAILURE Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org
