Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/93/

4 tests failed.
FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Expected numSlices=5 numReplicas=1 but found 
DocCollection(solrj_collection4//collections/solrj_collection4/state.json/23)={ 
  "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{    
   "range":"80000000-b332ffff",       "state":"active",       
"replicas":{"core_node2":{           
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node2/data/",
           "base_url":"http://127.0.0.1:43284/_qf/y";,           
"node_name":"127.0.0.1:43284__qf%2Fy",           "type":"NRT",           
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node2/data/tlog",
           "core":"solrj_collection4_shard1_replica_n1",           
"state":"active",           "leader":"true"}}},     "shard2":{       
"range":"b3330000-e665ffff",       "state":"active",       
"replicas":{"core_node5":{           
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node5/data/",
           "base_url":"http://127.0.0.1:40966/_qf/y";,           
"node_name":"127.0.0.1:40966__qf%2Fy",           "type":"NRT",           
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",
           "core":"solrj_collection4_shard2_replica_n3",           
"state":"active",           "leader":"true"}}},     "shard3":{       
"range":"e6660000-1998ffff",       "state":"active",       
"replicas":{"core_node7":{           
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node7/data/",
           "base_url":"http://127.0.0.1:43284/_qf/y";,           
"node_name":"127.0.0.1:43284__qf%2Fy",           "type":"NRT",           
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",
           "core":"solrj_collection4_shard3_replica_n4",           
"state":"active",           "leader":"true"}}},     "shard4":{       
"range":"19990000-4ccbffff",       "state":"active",       
"replicas":{"core_node9":{           
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node9/data/",
           "base_url":"http://127.0.0.1:38594/_qf/y";,           
"node_name":"127.0.0.1:38594__qf%2Fy",           "type":"NRT",           
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
           "core":"solrj_collection4_shard4_replica_n6",           
"state":"active",           "leader":"true"}}},     "shard5":{       
"range":"4ccc0000-7fffffff",       "state":"active",       
"replicas":{"core_node10":{           
"core":"solrj_collection4_shard5_replica_n8",           
"base_url":"http://127.0.0.1:40966/_qf/y";,           
"node_name":"127.0.0.1:40966__qf%2Fy",           "state":"down",           
"type":"NRT"}}}},   "router":{     "field":"text",     "name":"compositeId"},   
"maxShardsPerNode":"5",   "autoAddReplicas":"true",   "nrtReplicas":"1",   
"tlogReplicas":"0"} with /live_nodes: [127.0.0.1:38594__qf%2Fy, 
127.0.0.1:40966__qf%2Fy, 127.0.0.1:43284__qf%2Fy, 127.0.0.1:47041__qf%2Fy, 
127.0.0.1:48839__qf%2Fy]

Stack Trace:
java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found 
DocCollection(solrj_collection4//collections/solrj_collection4/state.json/23)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{"core_node2":{
          
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node2/data/",
          "base_url":"http://127.0.0.1:43284/_qf/y";,
          "node_name":"127.0.0.1:43284__qf%2Fy",
          "type":"NRT",
          
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node2/data/tlog",
          "core":"solrj_collection4_shard1_replica_n1",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{"core_node5":{
          
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node5/data/",
          "base_url":"http://127.0.0.1:40966/_qf/y";,
          "node_name":"127.0.0.1:40966__qf%2Fy",
          "type":"NRT",
          
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node5/data/tlog",
          "core":"solrj_collection4_shard2_replica_n3",
          "state":"active",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{"core_node7":{
          
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node7/data/",
          "base_url":"http://127.0.0.1:43284/_qf/y";,
          "node_name":"127.0.0.1:43284__qf%2Fy",
          "type":"NRT",
          
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node7/data/tlog",
          "core":"solrj_collection4_shard3_replica_n4",
          "state":"active",
          "leader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{"core_node9":{
          
"dataDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node9/data/",
          "base_url":"http://127.0.0.1:38594/_qf/y";,
          "node_name":"127.0.0.1:38594__qf%2Fy",
          "type":"NRT",
          
"ulogDir":"hdfs://localhost:47656/solr_hdfs_home/solrj_collection4/core_node9/data/tlog",
          "core":"solrj_collection4_shard4_replica_n6",
          "state":"active",
          "leader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{"core_node10":{
          "core":"solrj_collection4_shard5_replica_n8",
          "base_url":"http://127.0.0.1:40966/_qf/y";,
          "node_name":"127.0.0.1:40966__qf%2Fy",
          "state":"down",
          "type":"NRT"}}}},
  "router":{
    "field":"text",
    "name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"true",
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:38594__qf%2Fy, 
127.0.0.1:40966__qf%2Fy, 127.0.0.1:43284__qf%2Fy, 127.0.0.1:47041__qf%2Fy, 
127.0.0.1:48839__qf%2Fy]
        at 
__randomizedtesting.SeedInfo.seed([523B85F3603F19CE:DA6FBA29CEC37436]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:413)
        at 
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:272)
        at 
org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:143)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue

Error Message:
Error from server at https://127.0.0.1:43520/solr: Could not fully remove 
collection: tolerant

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:43520/solr: Could not fully remove collection: 
tolerant
        at 
__randomizedtesting.SeedInfo.seed([523B85F3603F19CE:60EB24F8659D78AF]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
        at 
org.apache.solr.cloud.TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue(TestDownShardTolerantSearch.java:54)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestRandomFlRTGCloud.testRandomizedUpdatesAndRTGs

Error Message:
Could not load collection from ZK: 
org.apache.solr.cloud.TestRandomFlRTGCloud_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: 
org.apache.solr.cloud.TestRandomFlRTGCloud_collection
        at 
__randomizedtesting.SeedInfo.seed([523B85F3603F19CE:ECE74D0FA79891E2]:0)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1122)
        at 
org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1205)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:848)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
        at 
org.apache.solr.cloud.TestRandomFlRTGCloud.assertRTG(TestRandomFlRTGCloud.java:396)
        at 
org.apache.solr.cloud.TestRandomFlRTGCloud.assertOneIter(TestRandomFlRTGCloud.java:266)
        at 
org.apache.solr.cloud.TestRandomFlRTGCloud.testRandomizedUpdatesAndRTGs(TestRandomFlRTGCloud.java:223)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: 
KeeperErrorCode = Session expired for 
/collections/org.apache.solr.cloud.TestRandomFlRTGCloud_collection/state.json
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
        at 
org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
        at 
org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1134)
        at 
org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1120)
        ... 49 more


FAILED:  org.apache.solr.update.CdcrUpdateLogTest.testSubReader

Error Message:


Stack Trace:
java.lang.AssertionError
        at 
__randomizedtesting.SeedInfo.seed([523B85F3603F19CE:1FE30F9010E8C81C]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertTrue(Assert.java:54)
        at 
org.apache.solr.update.CdcrUpdateLogTest.testSubReader(CdcrUpdateLogTest.java:583)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13108 lines...]
   [junit4] JVM J2: stdout was not empty, see: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8608049809614325084941.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] #
   [junit4] # There is insufficient memory for the Java Runtime Environment to 
continue.
   [junit4] # Native memory allocation (mmap) failed to map 65536 bytes for 
committing reserved memory.
   [junit4] # An error report file with more information is saved as:
   [junit4] # 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/hs_err_pid13719.log
   [junit4] [thread 140330072270592 also had an error]
   [junit4] #
   [junit4] # Compiler replay data is saved as:
   [junit4] # 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/replay_pid13719.log
   [junit4] <<< JVM J2: EOF ----

   [junit4] JVM J2: stderr was not empty, see: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8602427169084427390924.syserr
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] Java HotSpot(TM) 64-Bit Server VM warning: INFO: 
os::commit_memory(0x00007fa131100000, 65536, 1) failed; error='Cannot allocate 
memory' (errno=12)
   [junit4] Java HotSpot(TM) 64-Bit Server VM warning: INFO: 
os::commit_memory(0x00007fa1240be000, 12288, 0) failed; error='Cannot allocate 
memory' (errno=12)
   [junit4] <<< JVM J2: EOF ----

   [junit4] Suite: org.apache.solr.cloud.TestDownShardTolerantSearch
   [junit4]   2> 2865255 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/init-core-data-001
   [junit4]   2> 2865324 WARN  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=100 numCloses=100
   [junit4]   2> 2865416 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2865781 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 2865863 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001
   [junit4]   2> 2865882 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2865916 INFO  (Thread-1045) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2865916 INFO  (Thread-1045) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 2865930 ERROR (Thread-1045) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2866005 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:44053
   [junit4]   2> 2866269 INFO  (jetty-launcher-368-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2866272 INFO  (jetty-launcher-368-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2866278 INFO  (jetty-launcher-368-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3694841{/solr,null,AVAILABLE}
   [junit4]   2> 2866340 INFO  (jetty-launcher-368-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@5013d1b4{/solr,null,AVAILABLE}
   [junit4]   2> 2867969 INFO  (jetty-launcher-368-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@6865d590{SSL,[ssl, 
http/1.1]}{127.0.0.1:55301}
   [junit4]   2> 2867974 INFO  (jetty-launcher-368-thread-1) [    ] 
o.e.j.s.Server Started @2893662ms
   [junit4]   2> 2867974 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=55301}
   [junit4]   2> 2867974 ERROR (jetty-launcher-368-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2867974 INFO  (jetty-launcher-368-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@50302d79{SSL,[ssl, 
http/1.1]}{127.0.0.1:43520}
   [junit4]   2> 2867974 INFO  (jetty-launcher-368-thread-2) [    ] 
o.e.j.s.Server Started @2893662ms
   [junit4]   2> 2867974 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=43520}
   [junit4]   2> 2867974 ERROR (jetty-launcher-368-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2867981 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.2.0
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-11-28T10:44:46.927Z
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.2.0
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2867982 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-11-28T10:44:46.927Z
   [junit4]   2> 2868150 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2868244 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2868290 INFO  (jetty-launcher-368-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44053/solr
   [junit4]   2> 2868397 INFO  (jetty-launcher-368-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44053/solr
   [junit4]   2> 2869498 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2869499 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2869548 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:43520_solr
   [junit4]   2> 2869579 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:55301_solr
   [junit4]   2> 2869580 INFO  
(zkCallback-380-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2869581 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.Overseer Overseer 
(id=99081642630709254-127.0.0.1:55301_solr-n_0000000000) starting
   [junit4]   2> 2869689 INFO  
(zkCallback-379-thread-1-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2869837 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:55301_solr
   [junit4]   2> 2869864 INFO  
(zkCallback-379-thread-1-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2869936 INFO  
(zkCallback-380-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2872004 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_43520.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872041 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_55301.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872272 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_43520.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872272 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_43520.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872298 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_55301.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872298 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_55301.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2872338 INFO  (jetty-launcher-368-thread-2) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node2/.
   [junit4]   2> 2872339 INFO  (jetty-launcher-368-thread-1) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node1/.
   [junit4]   2> 2874647 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2874648 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44053/solr ready
   [junit4]   2> 2875113 INFO  
(TEST-TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue-seed#[523B85F3603F19CE])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting 
searchingShouldFailWithoutTolerantSearchSetToTrue
   [junit4]   2> 2876096 INFO  (qtp688232992-3378) [n:127.0.0.1:43520_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf&name=tolerant&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2876545 INFO  
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
tolerant
   [junit4]   2> 2876811 INFO  
(OverseerStateUpdate-99081642630709254-127.0.0.1:55301_solr-n_0000000000) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tolerant",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tolerant_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:43520/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2876953 INFO  
(OverseerStateUpdate-99081642630709254-127.0.0.1:55301_solr-n_0000000000) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tolerant",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"tolerant_shard2_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:55301/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2877520 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=tolerant_shard1_replica_n1&action=CREATE&numShards=2&collection=tolerant&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2877580 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 2877782 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tolerant_shard2_replica_n3&action=CREATE&numShards=2&collection=tolerant&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2877799 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 2877849 INFO  
(zkCallback-380-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2877856 INFO  
(zkCallback-379-thread-1-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2877860 INFO  
(zkCallback-379-thread-2-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2877992 INFO  
(zkCallback-379-thread-1-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2877993 INFO  
(zkCallback-380-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2878004 INFO  
(zkCallback-379-thread-2-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2878004 ERROR (jetty-launcher-368-thread-1-EventThread) 
[n:127.0.0.1:55301_solr    ] o.a.z.ClientCnxn Error while calling watcher 
   [junit4]   2> java.lang.OutOfMemoryError: unable to create new native thread
   [junit4]   2>        at java.lang.Thread.start0(Native Method)
   [junit4]   2>        at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:957)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1378)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:173)
   [junit4]   2>        at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2>        at 
org.apache.solr.common.cloud.SolrZkClient$1.process(SolrZkClient.java:262)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 2878005 INFO  
(zkCallback-380-thread-2-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2878870 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 2878886 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 2878929 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.s.IndexSchema [tolerant_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2878930 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.s.IndexSchema [tolerant_shard2_replica_n3] Schema name=minimal
   [junit4]   2> 2878943 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2878943 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.c.CoreContainer Creating SolrCore 'tolerant_shard2_replica_n3' using 
configuration from collection tolerant, trusted=true
   [junit4]   2> 2878960 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2878960 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'tolerant_shard1_replica_n1' using 
configuration from collection tolerant, trusted=true
   [junit4]   2> 2878965 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_43520.solr.core.tolerant.shard1.replica_n1' (registry 
'solr.core.tolerant.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2878965 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 2878965 INFO  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] o.a.s.c.SolrCore 
[[tolerant_shard1_replica_n1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node2/tolerant_shard1_replica_n1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node2/./tolerant_shard1_replica_n1/data/]
   [junit4]   2> 2878976 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_55301.solr.core.tolerant.shard2.replica_n3' (registry 
'solr.core.tolerant.shard2.replica_n3') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@5cc848f3
   [junit4]   2> 2878977 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 2878977 INFO  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] o.a.s.c.SolrCore 
[[tolerant_shard2_replica_n3] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node1/tolerant_shard2_replica_n3],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestDownShardTolerantSearch_523B85F3603F19CE-001/tempDir-001/node1/./tolerant_shard2_replica_n3/data/]
   [junit4]   2> 2879040 ERROR (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.a.s.s.HttpSolrCall null:java.lang.RuntimeException: 
java.lang.OutOfMemoryError: unable to create new native thread
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:663)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:531)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: unable to create new 
native thread
   [junit4]   2>        at java.lang.Thread.start0(Native Method)
   [junit4]   2>        at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3024)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:748)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1037)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2>        ... 29 more
   [junit4]   2> 
   [junit4]   2> 2879043 WARN  (qtp1670103853-3369) [n:127.0.0.1:55301_solr 
c:tolerant s:shard2 r:core_node4 x:tolerant_shard2_replica_n3] 
o.e.j.s.ServletHandler Error for /solr/admin/cores
   [junit4]   2> java.lang.OutOfMemoryError: unable to create new native thread
   [junit4]   2>        at java.lang.Thread.start0(Native Method)
   [junit4]   2>        at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3024)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:748)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1037)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2879041 ERROR (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.a.s.s.HttpSolrCall null:java.lang.RuntimeException: 
java.lang.OutOfMemoryError: unable to create new native thread
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:663)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:531)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.OutOfMemoryError: unable to create new 
native thread
   [junit4]   2>        at java.lang.Thread.start0(Native Method)
   [junit4]   2>        at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3024)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:748)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1037)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2>        ... 29 more
   [junit4]   2> 
   [junit4]   2> 2879045 WARN  (qtp688232992-3380) [n:127.0.0.1:43520_solr 
c:tolerant s:shard1 r:core_node2 x:tolerant_shard1_replica_n1] 
o.e.j.s.ServletHandler Error for /solr/admin/cores
   [junit4]   2> java.lang.OutOfMemoryError: unable to create new native thread
   [junit4]   2>        at java.lang.Thread.start0(Native Method)
   [junit4]   2>        at java.lang.Thread.start(Thread.java:717)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3024)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:748)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2>        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1037)
   [junit4]   2>        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2879056 ERROR 
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error 
from shard: https://127.0.0.1:55301/solr
   [junit4]   2> 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:55301/solr: Expected mime type 
application/octet-stream but got text/html. <html>
   [junit4]   2> <head>
   [junit4]   2> <meta http-equiv="Content-Type" 
content="text/html;charset=ISO-8859-1"/>
   [junit4]   2> <title>Error 500 </title>
   [junit4]   2> </head>
   [junit4]   2> <body>
   [junit4]   2> <h2>HTTP ERROR: 500</h2>
   [junit4]   2> <p>Problem accessing /solr/admin/cores. Reason:
   [junit4]   2> <pre>    java.lang.OutOfMemoryError: unable to create new 
native thread</pre></p>
   [junit4]   2> <hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 
9.3.20.v20170531</a><hr/>
   [junit4]   2> </body>
   [junit4]   2> </html>
   [junit4]   2> 
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2>        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2>        at 
org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2879056 ERROR 
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error 
from shard: https://127.0.0.1:43520/solr
   [junit4]   2> 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:43520/solr: Expected mime type 
application/octet-stream but got text/html. <html>
   [junit4]   2> <head>
   [junit4]   2> <meta http-equiv="Content-Type" 
content="text/html;charset=ISO-8859-1"/>
   [junit4]   2> <title>Error 500 </title>
   [junit4]   2> </head>
   [junit4]   2> <body>
   [junit4]   2> <h2>HTTP ERROR: 500</h2>
   [junit4]   2> <p>Problem accessing /solr/admin/cores. Reason:
   [junit4]   2> <pre>    java.lang.OutOfMemoryError: unable to create new 
native thread</pre></p>
   [junit4]   2> <hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 
9.3.20.v20170531</a><hr/>
   [junit4]   2> </body>
   [junit4]   2> </html>
   [junit4]   2> 
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]   2>        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2>        at 
org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2879056 ERROR 
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerCollectionMessageHandler Cleaning 
up collection [tolerant].
   [junit4]   2> 2879866 INFO  
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing 
Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 2879896 INFO  (qtp688232992-3374) [n:127.0.0.1:43520_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=tolerant_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=1
   [junit4]   2> 2879912 INFO  (qtp1670103853-3372) [n:127.0.0.1:55301_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=tolerant_shard2_replica_n3&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=3
   [junit4]   2> 2910014 INFO  
(zkCallback-380-thread-2-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDeleted 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2910014 INFO  
(zkCallback-380-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDeleted 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2910016 ERROR 
(OverseerThreadFactory-1065-thread-1-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: tolerant operation: create 
failed:org.apache.solr.common.SolrException: Could not fully remove collection: 
tolerant
   [junit4]   2>        at 
org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:109)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.cleanupCollection(OverseerCollectionMessageHandler.java:678)
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:302)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:232)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2910022 INFO  
(zkCallback-379-thread-2-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDeleted 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2910022 INFO  
(zkCallback-379-thread-1-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDeleted 
path:/collections/tolerant/state.json] for collection [tolerant] has occurred - 
updating... (live nodes size: [2])
   [junit4]   2> 2910175 INFO  (qtp688232992-3378) [n:127.0.0.1:43520_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf&name=tolerant&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2}
 status=500 QTime=34078
   [junit4]   2> 2910176 ERROR (qtp688232992-3378) [n:127.0.0.1:43520_solr    ] 
o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: Could not fully 
remove collection: tolerant
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:307)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2910177 INFO  
(TEST-TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue-seed#[523B85F3603F19CE])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending 
searchingShouldFailWithoutTolerantSearchSetToTrue
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestDownShardTolerantSearch 
-Dtests.method=searchingShouldFailWithoutTolerantSearchSetToTrue 
-Dtests.seed=523B85F3603F19CE -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=ar-SY -Dtests.timezone=Pacific/Ponape -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   35.5s J0 | 
TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue 
<<<
   [junit4]    > Throwable #1: 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at https://127.0.0.1:43520/solr: Could not fully remove collection: 
tolerant
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([523B85F3603F19CE:60EB24F8659D78AF]:0)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
   [junit4]    >        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    >        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    >        at 
org.apache.solr.cloud.TestDownShardTolerantSearch.searchingShouldFailWithoutTolerantSearchSetToTrue(TestDownShardTolerantSearch.java:54)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2910221 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x160023aa0c40008, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2910249 INFO  (jetty-closer-369-thread-1) [    ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@50302d79{SSL,[ssl, 
http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2910255 INFO  (jetty-closer-369-thread-2) [    ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@6865d590{SSL,[ssl, 
http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2910315 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1914838420
   [junit4]   2> 2910315 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, 
tag=null
   [junit4]   2> 2910316 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1412046965
   [junit4]   2> 2910316 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, 
tag=null
   [junit4]   2> 2910324 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@73ac8931: rootName = 
solr_55301, domain = solr.node, service url = null, agent id = null] for 
registry solr.node / com.codahale.metrics.MetricRegistry@14e05372
   [junit4]   2> 2910332 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@4582e3da: rootName = 
solr_43520, domain = solr.node, service url = null, agent id = null] for 
registry solr.node / com.codahale.metrics.MetricRegistry@4f3f7a97
   [junit4]   2> 2910346 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, 
tag=null
   [junit4]   2> 2910346 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@78dc3e31: rootName = 
solr_55301, domain = solr.jvm, service url = null, agent id = null] for 
registry solr.jvm / com.codahale.metrics.MetricRegistry@570d9826
   [junit4]   2> 2910360 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, 
tag=null
   [junit4]   2> 2910360 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@772fc171: rootName = 
solr_43520, domain = solr.jvm, service url = null, agent id = null] for 
registry solr.jvm / com.codahale.metrics.MetricRegistry@570d9826
   [junit4]   2> 2910364 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 2910364 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@27bce2b: rootName = 
solr_43520, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@5332161f
   [junit4]   2> 2910381 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 2910382 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@7272cbaa: rootName = 
solr_55301, domain = solr.jetty, service url = null, agent id = null] for 
registry solr.jetty / com.codahale.metrics.MetricRegistry@5332161f
   [junit4]   2> 2910385 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:55301_solr
   [junit4]   2> 2910386 INFO  
(zkCallback-380-thread-2-processing-n:127.0.0.1:55301_solr) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 2910387 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 2910400 INFO  (jetty-closer-369-thread-2) [    ] 
o.a.s.c.Overseer Overseer 
(id=99081642630709254-127.0.0.1:55301_solr-n_0000000000) closing
   [junit4]   2> 2910400 INFO  
(OverseerStateUpdate-99081642630709254-127.0.0.1:55301_solr-n_0000000000) 
[n:127.0.0.1:55301_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:55301_solr
   [junit4]   2> 2910409 WARN  
(OverseerAutoScalingTriggerThread-99081642630709254-127.0.0.1:55301_solr-n_0000000000)
 [n:127.0.0.1:55301_solr    ] o.a.s.c.a.OverseerTriggerThread 
OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2910412 INFO  
(zkCallback-379-thread-3-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 2910452 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.c.ZkController Remove node as live in 
ZooKeeper:/live_nodes/127.0.0.1:43520_solr
   [junit4]   2> 2910453 INFO  
(zkCallback-379-thread-3-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:43520_solr
   [junit4]   2> 2910490 INFO  (jetty-closer-369-thread-1) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 2910496 INFO  (jetty-closer-369-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@5013d1b4{/solr,null,UNAVAILABLE}
   [junit4]   2> 2910514 INFO  
(zkCallback-379-thread-2-processing-n:127.0.0.1:43520_solr) 
[n:127.0.0.1:43520_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 2912034 INFO  (jetty-closer-369-thread-1) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@3694841{/solr,null,UNAVAILABLE}
   [junit4]   2> 2912116 ERROR 
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2912117 INFO  
(SUITE-TestDownShardTolerantSearch-seed#[523B85F3603F19CE]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:44053 44053
   [junit4]   2> 2912239 INFO  (Thread-1045) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:44053 44053
   [junit4]   2> 2912239 WARN  (Thread-1045) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        3       /solr/aliases.json
   [junit4]   2>        2       /solr/security.json
   [junit4]   2>        2       /solr/collections/tolerant/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        7       /solr/collections/tolerant/state.json
   [junit4]   2>        3       /solr/clusterstate.json
   [junit4]   2>        3       /solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        3       /solr/live_nodes
   [junit4]   2>        3       /solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Nig

[...truncated too long message...]

ightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.2.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar
 com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8604394224732361570308.events
 
@/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8603357724846425214909.suites
 -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Forked process returned with 
error code: 1. Very likely a JVM crash.  See process stdout at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8608049809614325084941.sysout
 See process stderr at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20171128_095632_8602427169084427390924.syserr
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4]     at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]     at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:843:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:779:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:59:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build.xml:262:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/common-build.xml:556:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1512:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/common-build.xml:1038:
 At least one slave process threw an exception, first: Forked process returned 
with error code: 1. Very likely a JVM crash.  See process stdout at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20171128_095632_8324121800844277559615.sysout
 See process stderr at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20171128_095632_8322235683252346955190.syserr

Total time: 1023 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-NightlyTests-7.x #91 to compare, 
so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to