Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2104/

11 tests failed.
FAILED:  
org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest.testAsyncRequests

Error Message:
Could not load collection from ZK: testAsyncOperations

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: 
testAsyncOperations
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:373ADAE0E4FF7E90]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at 
org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at 
org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at 
org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:108)
        at 
org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest.testAsyncRequests(CollectionsAPIAsyncDistributedZkTest.java:128)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for 
/collections/testAsyncOperations/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 43 more


FAILED:  
org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Error from server at https://127.0.0.1:38102/solr: KeeperErrorCode = NoNode for 
/overseer/collection-queue-work/qnr-0000000016

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:38102/solr: KeeperErrorCode = NoNode for 
/overseer/collection-queue-work/qnr-0000000016
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:862E0EC5EEAEFFBF]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
        at 
org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:135)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1268)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest

Error Message:
expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:A78E0712067377C0]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest(TestCloudRecovery.java:132)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test

Error Message:
expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
        at 
org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestPullReplica.testRemoveAllWriterReplicas

Error Message:
Replica core_node4 not up to date after 10 seconds expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: Replica core_node4 not up to date after 10 seconds 
expected:<1> but was:<0>
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:40707A6C56A9D9E6]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at 
org.apache.solr.cloud.TestPullReplica.waitForNumDocsInAllReplicas(TestPullReplica.java:510)
        at 
org.apache.solr.cloud.TestPullReplica.waitForNumDocsInAllReplicas(TestPullReplica.java:501)
        at 
org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:385)
        at 
org.apache.solr.cloud.TestPullReplica.testRemoveAllWriterReplicas(TestPullReplica.java:286)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerConcurrent.test

Error Message:
Could not load collection from ZK: collection1

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: 
collection1
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at 
org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at 
org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.getTotalReplicas(AbstractFullDistribZkTestBase.java:488)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:441)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for /collections/collection1/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 38 more


FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
inplace_updatable_float didn't match for replica at client: 
https://127.0.0.1:37233/collection1 expected:<105.0> but was:<1.0>

Stack Trace:
java.lang.AssertionError: inplace_updatable_float didn't match for replica at 
client: https://127.0.0.1:37233/collection1 expected:<105.0> but was:<1.0>
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:980)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.outOfOrderUpdatesIndividualReplicaTest(TestInPlaceUpdatesDistrib.java:643)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:144)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.common.cloud.TestCollectionStateWatchers.testSimpleCollectionWatch

Error Message:
Could not load collection from ZK: testcollection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: 
testcollection
        at 
__randomizedtesting.SeedInfo.seed([641C592B70F0F21E:3927965B37FD6D20]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1115)
        at 
org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:648)
        at 
org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
        at 
org.apache.solr.common.cloud.ZkStateReader.registerCollectionStateWatcher(ZkStateReader.java:1233)
        at 
org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1263)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
        at 
org.apache.solr.common.cloud.TestCollectionStateWatchers.testSimpleCollectionWatch(TestCollectionStateWatchers.java:117)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for /collections/testcollection/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at 
org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1127)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1113)
        ... 45 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; 
[core_node4:{"core":"c8n_1x3_lf_shard1_replica_t1","base_url":"http://127.0.0.1:60090","node_name":"127.0.0.1:60090_","state":"active","type":"TLOG","leader":"true"}];
 clusterState: 
DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/16)={   
"pullReplicas":"0",   "replicationFactor":"1",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node4":{           "core":"c8n_1x3_lf_shard1_replica_t1",           
"base_url":"http://127.0.0.1:60090";,           "node_name":"127.0.0.1:60090_",  
         "state":"active",           "type":"TLOG",           "leader":"true"}, 
        "core_node5":{           "core":"c8n_1x3_lf_shard1_replica_t3",         
  "base_url":"http://127.0.0.1:51606";,           
"node_name":"127.0.0.1:51606_",           "state":"down",           
"type":"TLOG"},         "core_node6":{           "state":"down",           
"base_url":"http://127.0.0.1:58188";,           
"core":"c8n_1x3_lf_shard1_replica_t2",           
"node_name":"127.0.0.1:58188_",           "type":"TLOG"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"1",   
"autoAddReplicas":"false",   "nrtReplicas":"0",   "tlogReplicas":"3"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 
1; 
[core_node4:{"core":"c8n_1x3_lf_shard1_replica_t1","base_url":"http://127.0.0.1:60090","node_name":"127.0.0.1:60090_","state":"active","type":"TLOG","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/16)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"c8n_1x3_lf_shard1_replica_t1",
          "base_url":"http://127.0.0.1:60090";,
          "node_name":"127.0.0.1:60090_",
          "state":"active",
          "type":"TLOG",
          "leader":"true"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_t3",
          "base_url":"http://127.0.0.1:51606";,
          "node_name":"127.0.0.1:51606_",
          "state":"down",
          "type":"TLOG"},
        "core_node6":{
          "state":"down",
          "base_url":"http://127.0.0.1:58188";,
          "core":"c8n_1x3_lf_shard1_replica_t2",
          "node_name":"127.0.0.1:58188_",
          "type":"TLOG"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"0",
  "tlogReplicas":"3"}
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! 
ClusterState: {   "collMinRf_1x3":{     "pullReplicas":"0",     
"replicationFactor":"1",     "shards":{"shard1":{         
"range":"80000000-7fffffff",         "state":"active",         "replicas":{     
      "core_node4":{             "core":"collMinRf_1x3_shard1_replica_t1",      
       "base_url":"http://127.0.0.1:54417/ui_sk/gi";,             
"node_name":"127.0.0.1:54417_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG",             "leader":"true"},           "core_node5":{       
      "core":"collMinRf_1x3_shard1_replica_t2",             
"base_url":"http://127.0.0.1:58034/ui_sk/gi";,             
"node_name":"127.0.0.1:58034_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG"},           "core_node6":{             
"core":"collMinRf_1x3_shard1_replica_t3",             
"base_url":"http://127.0.0.1:50277/ui_sk/gi";,             
"node_name":"127.0.0.1:50277_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG"}}}},     "router":{"name":"compositeId"},     
"maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"0",   
  "tlogReplicas":"3"},   "collection1":{     "pullReplicas":"0",     
"replicationFactor":"1",     "shards":{       "shard1":{         
"range":"80000000-ffffffff",         "state":"active",         
"replicas":{"core_node44":{             
"core":"collection1_shard1_replica_t43",             
"base_url":"http://127.0.0.1:58034/ui_sk/gi";,             
"node_name":"127.0.0.1:58034_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG",             "leader":"true"}}},       "shard2":{         
"range":"0-7fffffff",         "state":"active",         "replicas":{           
"core_node42":{             "core":"collection1_shard2_replica_t41",            
 "base_url":"http://127.0.0.1:56924/ui_sk/gi";,             
"node_name":"127.0.0.1:56924_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG",             "leader":"true"},           "core_node46":{      
       "core":"collection1_shard2_replica_t45",             
"base_url":"http://127.0.0.1:54417/ui_sk/gi";,             
"node_name":"127.0.0.1:54417_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG"}}}},     "router":{"name":"compositeId"},     
"maxShardsPerNode":"1",     "autoAddReplicas":"false",     "nrtReplicas":"1",   
  "tlogReplicas":"0"},   "control_collection":{     "pullReplicas":"0",     
"replicationFactor":"1",     "shards":{"shard1":{         
"range":"80000000-7fffffff",         "state":"active",         
"replicas":{"core_node2":{             
"core":"control_collection_shard1_replica_n1",             
"base_url":"http://127.0.0.1:50277/ui_sk/gi";,             
"node_name":"127.0.0.1:50277_ui_sk%2Fgi",             "state":"active",         
    "type":"NRT",             "leader":"true"}}}},     
"router":{"name":"compositeId"},     "maxShardsPerNode":"1",     
"autoAddReplicas":"false",     "nrtReplicas":"1",     "tlogReplicas":"0"},   
"c8n_1x2":{     "pullReplicas":"0",     "replicationFactor":"1",     
"shards":{"shard1":{         "range":"80000000-7fffffff",         
"state":"active",         "replicas":{           "core_node3":{             
"state":"down",             "base_url":"http://127.0.0.1:54417/ui_sk/gi";,       
      "core":"c8n_1x2_shard1_replica_t1",             
"node_name":"127.0.0.1:54417_ui_sk%2Fgi",             "type":"TLOG"},           
"core_node4":{             "core":"c8n_1x2_shard1_replica_t2",             
"base_url":"http://127.0.0.1:50277/ui_sk/gi";,             
"node_name":"127.0.0.1:50277_ui_sk%2Fgi",             "state":"active",         
    "type":"TLOG",             "leader":"true"}}}},     
"router":{"name":"compositeId"},     "maxShardsPerNode":"1",     
"autoAddReplicas":"false",     "nrtReplicas":"0",     "tlogReplicas":"2"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 
come up within 90000 ms! ClusterState: {
  "collMinRf_1x3":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collMinRf_1x3_shard1_replica_t1",
            "base_url":"http://127.0.0.1:54417/ui_sk/gi";,
            "node_name":"127.0.0.1:54417_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node5":{
            "core":"collMinRf_1x3_shard1_replica_t2",
            "base_url":"http://127.0.0.1:58034/ui_sk/gi";,
            "node_name":"127.0.0.1:58034_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG"},
          "core_node6":{
            "core":"collMinRf_1x3_shard1_replica_t3",
            "base_url":"http://127.0.0.1:50277/ui_sk/gi";,
            "node_name":"127.0.0.1:50277_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"0",
    "tlogReplicas":"3"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node44":{
            "core":"collection1_shard1_replica_t43",
            "base_url":"http://127.0.0.1:58034/ui_sk/gi";,
            "node_name":"127.0.0.1:58034_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node42":{
            "core":"collection1_shard2_replica_t41",
            "base_url":"http://127.0.0.1:56924/ui_sk/gi";,
            "node_name":"127.0.0.1:56924_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG",
            "leader":"true"},
          "core_node46":{
            "core":"collection1_shard2_replica_t45",
            "base_url":"http://127.0.0.1:54417/ui_sk/gi";,
            "node_name":"127.0.0.1:54417_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "base_url":"http://127.0.0.1:50277/ui_sk/gi";,
            "node_name":"127.0.0.1:50277_ui_sk%2Fgi",
            "state":"active",
            "type":"NRT",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x2":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "state":"down",
            "base_url":"http://127.0.0.1:54417/ui_sk/gi";,
            "core":"c8n_1x2_shard1_replica_t1",
            "node_name":"127.0.0.1:54417_ui_sk%2Fgi",
            "type":"TLOG"},
          "core_node4":{
            "core":"c8n_1x2_shard1_replica_t2",
            "base_url":"http://127.0.0.1:50277/ui_sk/gi";,
            "node_name":"127.0.0.1:50277_ui_sk%2Fgi",
            "state":"active",
            "type":"TLOG",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "nrtReplicas":"0",
    "tlogReplicas":"2"}}
        at 
__randomizedtesting.SeedInfo.seed([D37EE6574257304F:5B2AD98DECAB5DB7]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
        at 
org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:324)
        at 
org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11732 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestPullReplica
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/init-core-data-001
   [junit4]   2> 783050 WARN  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
   [junit4]   2> 783050 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 783051 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 783051 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 783134 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001
   [junit4]   2> 783137 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 783137 INFO  (Thread-274) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 783137 INFO  (Thread-274) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 783226 ERROR (Thread-274) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 783242 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:53801
   [junit4]   2> 783559 INFO  (jetty-launcher-179-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 783670 INFO  (jetty-launcher-179-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 784010 INFO  (jetty-launcher-179-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3fd319f5{/solr,null,AVAILABLE}
   [junit4]   2> 784021 INFO  (jetty-launcher-179-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@4b6627e1{/solr,null,AVAILABLE}
   [junit4]   2> 784022 INFO  (jetty-launcher-179-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@589712e8{HTTP/1.1,[http/1.1]}{127.0.0.1:46528}
   [junit4]   2> 784022 INFO  (jetty-launcher-179-thread-2) [    ] 
o.e.j.s.Server Started @802361ms
   [junit4]   2> 784022 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=46528}
   [junit4]   2> 784023 ERROR (jetty-launcher-179-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-09-24T00:42:23.999Z
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@462779e2{HTTP/1.1,[http/1.1]}{127.0.0.1:49079}
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.e.j.s.Server Started @802362ms
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=49079}
   [junit4]   2> 784023 ERROR (jetty-launcher-179-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
8.0.0
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 784023 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-09-24T00:42:23.999Z
   [junit4]   2> 784226 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 784312 INFO  (jetty-launcher-179-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53801/solr
   [junit4]   2> 784419 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 784492 INFO  (jetty-launcher-179-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53801/solr
   [junit4]   2> 785121 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 785122 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:46528_solr
   [junit4]   2> 785123 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.Overseer Overseer 
(id=98711223871012870-127.0.0.1:46528_solr-n_0000000000) starting
   [junit4]   2> 785180 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:46528_solr
   [junit4]   2> 785274 INFO  
(OverseerStateUpdate-98711223871012870-127.0.0.1:46528_solr-n_0000000000) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 785414 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 785419 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 785430 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:49079_solr
   [junit4]   2> 785434 INFO  
(zkCallback-192-thread-1-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 785577 INFO  
(zkCallback-187-thread-1-processing-n:127.0.0.1:49079_solr) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 788127 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46528.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788133 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46528.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788134 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46528.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788135 INFO  (jetty-launcher-179-thread-2) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node2/.
   [junit4]   2> 788408 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_49079.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788514 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_49079.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788515 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_49079.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 788537 INFO  (jetty-launcher-179-thread-1) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node1/.
   [junit4]   2> 789330 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 789333 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53801/solr ready
   [junit4]   2> 789368 INFO  
(SUITE-TestPullReplica-seed#[D37EE6574257304F]-worker) [    ] 
o.a.s.c.TestPullReplica Using legacyCloud?: false
   [junit4]   2> 789402 INFO  (qtp840288847-1315) [n:127.0.0.1:46528_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params 
val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and 
sendToOCPQueue=true
   [junit4]   2> 789413 INFO  (qtp840288847-1315) [n:127.0.0.1:46528_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} 
status=0 QTime=11
   [junit4]   2> 789486 INFO  
(TEST-TestPullReplica.testKillPullReplica-seed#[D37EE6574257304F]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testKillPullReplica
   [junit4]   2> 789488 INFO  (qtp840288847-1316) [n:127.0.0.1:46528_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
pullReplicas=1&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_kill_pull_replica&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 789529 INFO  
(OverseerThreadFactory-570-thread-1-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
pull_replica_test_kill_pull_replica
   [junit4]   2> 789778 INFO  
(OverseerStateUpdate-98711223871012870-127.0.0.1:46528_solr-n_0000000000) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_kill_pull_replica",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   
"core":"pull_replica_test_kill_pull_replica_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:46528/solr";,
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 789779 INFO  
(OverseerStateUpdate-98711223871012870-127.0.0.1:46528_solr-n_0000000000) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_kill_pull_replica",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   
"core":"pull_replica_test_kill_pull_replica_shard1_replica_p2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:49079/solr";,
   [junit4]   2>   "type":"PULL"} 
   [junit4]   2> 790322 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_pull_replica_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_kill_pull_replica&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 790323 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 790492 INFO  
(zkCallback-192-thread-1-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791015 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_pull_replica_shard1_replica_p2&action=CREATE&numShards=1&collection=pull_replica_test_kill_pull_replica&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 791046 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 790969 INFO  
(zkCallback-192-thread-2-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791178 INFO  
(zkCallback-187-thread-1-processing-n:127.0.0.1:49079_solr) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791194 INFO  
(zkCallback-187-thread-2-processing-n:127.0.0.1:49079_solr) 
[n:127.0.0.1:49079_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791248 INFO  
(zkCallback-192-thread-2-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791249 INFO  
(zkCallback-192-thread-3-processing-n:127.0.0.1:46528_solr) 
[n:127.0.0.1:46528_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/pull_replica_test_kill_pull_replica/state.json] for 
collection [pull_replica_test_kill_pull_replica] has occurred - updating... 
(live nodes size: [2])
   [junit4]   2> 791990 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 792144 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.s.IndexSchema 
[pull_replica_test_kill_pull_replica_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 792183 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 792183 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.c.CoreContainer 
Creating SolrCore 'pull_replica_test_kill_pull_replica_shard1_replica_n1' using 
configuration from collection pull_replica_test_kill_pull_replica, trusted=true
   [junit4]   2> 792183 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_46528.solr.core.pull_replica_test_kill_pull_replica.shard1.replica_n1' 
(registry 'solr.core.pull_replica_test_kill_pull_replica.shard1.replica_n1') 
enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 792183 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 792183 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.c.SolrCore 
[[pull_replica_test_kill_pull_replica_shard1_replica_n1] ] Opening new SolrCore 
at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node2/pull_replica_test_kill_pull_replica_shard1_replica_n1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node2/./pull_replica_test_kill_pull_replica_shard1_replica_n1/data/]
   [junit4]   2> 792568 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 792632 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.s.IndexSchema 
[pull_replica_test_kill_pull_replica_shard1_replica_p2] Schema name=minimal
   [junit4]   2> 792690 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 792690 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.CoreContainer 
Creating SolrCore 'pull_replica_test_kill_pull_replica_shard1_replica_p2' using 
configuration from collection pull_replica_test_kill_pull_replica, trusted=true
   [junit4]   2> 792690 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_49079.solr.core.pull_replica_test_kill_pull_replica.shard1.replica_p2' 
(registry 'solr.core.pull_replica_test_kill_pull_replica.shard1.replica_p2') 
enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@59b90e06
   [junit4]   2> 792691 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 792691 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.SolrCore 
[[pull_replica_test_kill_pull_replica_shard1_replica_p2] ] Opening new SolrCore 
at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node1/pull_replica_test_kill_pull_replica_shard1_replica_p2],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_D37EE6574257304F-001/tempDir-001/node1/./pull_replica_test_kill_pull_replica_shard1_replica_p2/data/]
   [junit4]   2> 793257 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 793257 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 793355 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 793355 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 793453 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 793453 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 793462 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1f3b0647[pull_replica_test_kill_pull_replica_shard1_replica_n1] main]
   [junit4]   2> 793462 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@799682b7[pull_replica_test_kill_pull_replica_shard1_replica_p2] main]
   [junit4]   2> 793466 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 793466 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 793467 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 793470 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 793471 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 793472 INFO  (qtp840288847-1313) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 793536 INFO  
(searcherExecutor-575-thread-1-processing-n:127.0.0.1:46528_solr 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1 s:shard1 
c:pull_replica_test_kill_pull_replica r:core_node3) [n:127.0.0.1:46528_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node3 
x:pull_replica_test_kill_pull_replica_shard1_replica_n1] o.a.s.c.SolrCore 
[pull_replica_test_kill_pull_replica_shard1_replica_n1] Registered new searcher 
Searcher@1f3b0647[pull_replica_test_kill_pull_replica_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 793537 INFO  
(searcherExecutor-576-thread-1-processing-n:127.0.0.1:49079_solr 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2 s:shard1 
c:pull_replica_test_kill_pull_replica r:core_node4) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.SolrCore 
[pull_replica_test_kill_pull_replica_shard1_replica_p2] Registered new searcher 
Searcher@799682b7[pull_replica_test_kill_pull_replica_shard1_replica_p2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 793538 INFO  (qtp440333781-1306) [n:127.0.0.1:49079_solr 
c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 
x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.c.ZkController 
pull_replica_test_kill_pull_replica_shard1_replica_p2 starting background 
replication from leader
   [junit4]   2> 793538 INFO  (qtp440333781-1306) [n:127.0.0.1:49079

[...truncated too long message...]

tateformat1_shard1_replica_n1] o.a.s.c.SolrCore 
[stateformat1_shard1_replica_n1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@5ded8c51
   [junit4]   2> 186927 INFO  (coreCloseExecutor-129-thread-2) 
[n:127.0.0.1:38516_solr c:testcollection s:shard4 r:core_node8 
x:testcollection_shard4_replica_n7] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.testcollection.shard4.leader, 
tag=40979878
   [junit4]   2> 186957 INFO  (coreCloseExecutor-129-thread-1) 
[n:127.0.0.1:38516_solr c:falsepredicate s:shard4 r:core_node8 
x:falsepredicate_shard4_replica_n6] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.falsepredicate.shard4.leader, 
tag=1106305326
   [junit4]   2> 186986 INFO  (coreCloseExecutor-131-thread-4) 
[n:127.0.0.1:59494_solr c:falsepredicate s:shard1 r:core_node3 
x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.core.falsepredicate.shard1.replica_n1, 
tag=1350282811
   [junit4]   2> 186986 INFO  (coreCloseExecutor-131-thread-4) 
[n:127.0.0.1:59494_solr c:falsepredicate s:shard1 r:core_node3 
x:falsepredicate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@58dae8ee: rootName = 
solr_59494, domain = solr.core.falsepredicate.shard1.replica_n1, service url = 
null, agent id = null] for registry solr.core.falsepredicate.shard1.replica_n1 
/ com.codahale.metrics.MetricRegistry@4fcef3fa
   [junit4]   2> 187009 INFO  (coreCloseExecutor-130-thread-1) 
[n:127.0.0.1:45348_solr c:testcollection s:shard1 r:core_node2 
x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.testcollection.shard1.leader, 
tag=1412877550
   [junit4]   2> 187009 INFO  (coreCloseExecutor-130-thread-3) 
[n:127.0.0.1:45348_solr c:currentstate s:shard1 r:core_node2 
x:currentstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.currentstate.shard1.leader, 
tag=1456421059
   [junit4]   2> 187677 INFO  
(zkCallback-24-thread-1-processing-n:127.0.0.1:59494_solr) 
[n:127.0.0.1:59494_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 187677 INFO  
(zkCallback-21-thread-2-processing-n:127.0.0.1:45348_solr) 
[n:127.0.0.1:45348_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 187677 INFO  
(zkCallback-48-thread-4-processing-n:127.0.0.1:38516_solr) 
[n:127.0.0.1:38516_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (4) -> (3)
   [junit4]   2> 187719 INFO  (coreCloseExecutor-131-thread-2) 
[n:127.0.0.1:59494_solr c:waitforstate s:shard1 r:core_node2 
x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.waitforstate.shard1.leader, tag=320046093
   [junit4]   2> 187719 INFO  (coreCloseExecutor-131-thread-4) 
[n:127.0.0.1:59494_solr c:falsepredicate s:shard1 r:core_node3 
x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.falsepredicate.shard1.leader, 
tag=1350282811
   [junit4]   2> 187787 INFO  (coreCloseExecutor-131-thread-5) 
[n:127.0.0.1:59494_solr c:stateformat1 s:shard1 r:core_node2 
x:stateformat1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.core.stateformat1.shard1.replica_n1, tag=1575849041
   [junit4]   2> 187787 INFO  (coreCloseExecutor-131-thread-5) 
[n:127.0.0.1:59494_solr c:stateformat1 s:shard1 r:core_node2 
x:stateformat1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@3a3fd59c: rootName = 
solr_59494, domain = solr.core.stateformat1.shard1.replica_n1, service url = 
null, agent id = null] for registry solr.core.stateformat1.shard1.replica_n1 / 
com.codahale.metrics.MetricRegistry@5e2df3b
   [junit4]   2> 188227 INFO  
(zkCallback-24-thread-1-processing-n:127.0.0.1:59494_solr) 
[n:127.0.0.1:59494_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 188227 INFO  
(zkCallback-21-thread-2-processing-n:127.0.0.1:45348_solr) 
[n:127.0.0.1:45348_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (2)
   [junit4]   2> 188228 INFO  (coreCloseExecutor-131-thread-5) 
[n:127.0.0.1:59494_solr c:stateformat1 s:shard1 r:core_node2 
x:stateformat1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric 
reporters for registry=solr.collection.stateformat1.shard1.leader, 
tag=1575849041
   [junit4]   2> 188229 INFO  (coreCloseExecutor-131-thread-3) 
[n:127.0.0.1:59494_solr c:delayed s:shard1 r:core_node2 
x:delayed_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters 
for registry=solr.collection.delayed.shard1.leader, tag=204287829
   [junit4]   2> 188442 INFO  (jetty-closer-3-thread-1) [    ] o.a.s.c.Overseer 
Overseer (id=98711908197203977-127.0.0.1:45348_solr-n_0000000000) closing
   [junit4]   2> 188442 INFO  
(OverseerStateUpdate-98711908197203977-127.0.0.1:45348_solr-n_0000000000) 
[n:127.0.0.1:45348_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:45348_solr
   [junit4]   2> 189273 WARN  
(zkCallback-42-thread-2-processing-n:127.0.0.1:57873_solr) 
[n:127.0.0.1:57873_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 190268 WARN  
(zkCallback-48-thread-2-processing-n:127.0.0.1:38516_solr) 
[n:127.0.0.1:38516_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 190471 INFO  (jetty-closer-3-thread-4) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@5ea3ad46{/solr,null,UNAVAILABLE}
   [junit4]   2> 190492 INFO  (jetty-closer-3-thread-3) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1c420d7b{/solr,null,UNAVAILABLE}
   [junit4]   2> 190494 INFO  
(zkCallback-24-thread-3-processing-n:127.0.0.1:59494_solr) 
[n:127.0.0.1:59494_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:59494_solr
   [junit4]   2> 190494 INFO  
(zkCallback-24-thread-1-processing-n:127.0.0.1:59494_solr) 
[n:127.0.0.1:59494_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 192001 WARN  
(zkCallback-21-thread-2-processing-n:127.0.0.1:45348_solr) 
[n:127.0.0.1:45348_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 192002 INFO  (jetty-closer-3-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@47bcbaf8{/solr,null,UNAVAILABLE}
   [junit4]   2> 192029 WARN  
(zkCallback-24-thread-3-processing-n:127.0.0.1:59494_solr) 
[n:127.0.0.1:59494_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 192029 INFO  (jetty-closer-3-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@53531fe1{/solr,null,UNAVAILABLE}
   [junit4]   2> 192046 ERROR 
(SUITE-TestCollectionStateWatchers-seed#[641C592B70F0F21E]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 192047 INFO  
(SUITE-TestCollectionStateWatchers-seed#[641C592B70F0F21E]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:41683 41683
   [junit4]   2> 192219 INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer connecting 
to 127.0.0.1:41683 41683
   [junit4]   2> 192223 WARN  (Thread-1) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        8       /solr/aliases.json
   [junit4]   2>        8       /solr/clusterprops.json
   [junit4]   2>        6       /solr/security.json
   [junit4]   2>        6       /solr/configs/config
   [junit4]   2>        3       /solr/collections/stateformat1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        79      /solr/collections/testcollection/state.json
   [junit4]   2>        73      /solr/collections/falsepredicate/state.json
   [junit4]   2>        18      /solr/collections/waitforstate/state.json
   [junit4]   2>        8       /solr/collections/delayed/state.json
   [junit4]   2>        8       /solr/clusterstate.json
   [junit4]   2>        7       /solr/collections/tobedeleted/state.json
   [junit4]   2>        5       /solr/collections/stateformat1/state.json
   [junit4]   2>        3       
/solr/overseer_elect/election/98711908197203980-127.0.0.1:59494_solr-n_0000000001
   [junit4]   2>        2       /solr/collections/currentstate/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        8       /solr/collections
   [junit4]   2>        6       /solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-solrj/test/J1/temp/solr.common.cloud.TestCollectionStateWatchers_641C592B70F0F21E-001
   [junit4]   2> Sep 24, 2017 3:39:32 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1271, maxMBSortInHeap=5.373536901245038, 
sim=RandomSimilarity(queryNorm=false): {}, locale=da-DK, timezone=Etc/GMT+11
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=193536672,total=285736960
   [junit4]   2> NOTE: All tests run in this JVM: [TestCollectionStateWatchers]
   [junit4] Completed [21/145 (1!)] on J1 in 202.63s, 9 tests, 1 error <<< 
FAILURES!

[...truncated 46056 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to