Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1825/

6 tests failed.
FAILED:  
org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Failed while waiting for active collection Timeout waiting to see state for 
collection=awhollynewcollection_0 
:DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-b332ffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_0_shard1_replica_n1", 
          "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node5":{      
     "core":"awhollynewcollection_0_shard1_replica_n2",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node11":{     
      "core":"awhollynewcollection_0_shard1_replica_n4",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node13":{     
      "core":"awhollynewcollection_0_shard1_replica_n6",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard2":{       
"range":"b3330000-e665ffff",       "state":"active",       "replicas":{         
"core_node15":{           "core":"awhollynewcollection_0_shard2_replica_n7",    
       "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node16":{     
      "core":"awhollynewcollection_0_shard2_replica_n8",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node18":{     
      "core":"awhollynewcollection_0_shard2_replica_n9",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node19":{     
      "core":"awhollynewcollection_0_shard2_replica_n10",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"e6660000-1998ffff",       "state":"active",       "replicas":{         
"core_node20":{           "core":"awhollynewcollection_0_shard3_replica_n12",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_0_shard3_replica_n14",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node30":{     
      "core":"awhollynewcollection_0_shard3_replica_n17",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_0_shard3_replica_n21",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"19990000-4ccbffff",       "state":"active",       "replicas":{         
"core_node33":{           "core":"awhollynewcollection_0_shard4_replica_n22",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node34":{     
      "core":"awhollynewcollection_0_shard4_replica_n23",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node35":{     
      "core":"awhollynewcollection_0_shard4_replica_n24",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node36":{     
      "core":"awhollynewcollection_0_shard4_replica_n25",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard5":{       
"range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{         
"core_node37":{           "core":"awhollynewcollection_0_shard5_replica_n26",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node38":{     
      "core":"awhollynewcollection_0_shard5_replica_n27",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node39":{     
      "core":"awhollynewcollection_0_shard5_replica_n29",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node40":{     
      "core":"awhollynewcollection_0_shard5_replica_n31",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"6",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"} Live 
Nodes: [127.0.0.1:34549_solr, 127.0.0.1:40912_solr, 127.0.0.1:41277_solr, 
127.0.0.1:43941_solr] Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-b332ffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_0_shard1_replica_n1", 
          "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node5":{      
     "core":"awhollynewcollection_0_shard1_replica_n2",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node11":{     
      "core":"awhollynewcollection_0_shard1_replica_n4",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node13":{     
      "core":"awhollynewcollection_0_shard1_replica_n6",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard2":{       
"range":"b3330000-e665ffff",       "state":"active",       "replicas":{         
"core_node15":{           "core":"awhollynewcollection_0_shard2_replica_n7",    
       "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node16":{     
      "core":"awhollynewcollection_0_shard2_replica_n8",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node18":{     
      "core":"awhollynewcollection_0_shard2_replica_n9",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node19":{     
      "core":"awhollynewcollection_0_shard2_replica_n10",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"e6660000-1998ffff",       "state":"active",       "replicas":{         
"core_node20":{           "core":"awhollynewcollection_0_shard3_replica_n12",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_0_shard3_replica_n14",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node30":{     
      "core":"awhollynewcollection_0_shard3_replica_n17",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_0_shard3_replica_n21",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"19990000-4ccbffff",       "state":"active",       "replicas":{         
"core_node33":{           "core":"awhollynewcollection_0_shard4_replica_n22",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node34":{     
      "core":"awhollynewcollection_0_shard4_replica_n23",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node35":{     
      "core":"awhollynewcollection_0_shard4_replica_n24",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node36":{     
      "core":"awhollynewcollection_0_shard4_replica_n25",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard5":{       
"range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{         
"core_node37":{           "core":"awhollynewcollection_0_shard5_replica_n26",   
        "base_url":"http://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node38":{     
      "core":"awhollynewcollection_0_shard5_replica_n27",           
"base_url":"http://127.0.0.1:34549/solr";,           
"node_name":"127.0.0.1:34549_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node39":{     
      "core":"awhollynewcollection_0_shard5_replica_n29",           
"base_url":"http://127.0.0.1:41277/solr";,           
"node_name":"127.0.0.1:41277_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node40":{     
      "core":"awhollynewcollection_0_shard5_replica_n31",           
"base_url":"http://127.0.0.1:43941/solr";,           
"node_name":"127.0.0.1:43941_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"6",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_0 
:DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard1_replica_n1",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"awhollynewcollection_0_shard1_replica_n2",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n4",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard1_replica_n6",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n7",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node16":{
          "core":"awhollynewcollection_0_shard2_replica_n8",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node18":{
          "core":"awhollynewcollection_0_shard2_replica_n9",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard2_replica_n10",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node20":{
          "core":"awhollynewcollection_0_shard3_replica_n12",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node30":{
          "core":"awhollynewcollection_0_shard3_replica_n17",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_0_shard3_replica_n21",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node33":{
          "core":"awhollynewcollection_0_shard4_replica_n22",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node34":{
          "core":"awhollynewcollection_0_shard4_replica_n23",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n24",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node36":{
          "core":"awhollynewcollection_0_shard4_replica_n25",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node37":{
          "core":"awhollynewcollection_0_shard5_replica_n26",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node38":{
          "core":"awhollynewcollection_0_shard5_replica_n27",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard5_replica_n29",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard5_replica_n31",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"6",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34549_solr, 127.0.0.1:40912_solr, 127.0.0.1:41277_solr, 
127.0.0.1:43941_solr]
Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard1_replica_n1",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"awhollynewcollection_0_shard1_replica_n2",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n4",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard1_replica_n6",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n7",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node16":{
          "core":"awhollynewcollection_0_shard2_replica_n8",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node18":{
          "core":"awhollynewcollection_0_shard2_replica_n9",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard2_replica_n10",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node20":{
          "core":"awhollynewcollection_0_shard3_replica_n12",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node30":{
          "core":"awhollynewcollection_0_shard3_replica_n17",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_0_shard3_replica_n21",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node33":{
          "core":"awhollynewcollection_0_shard4_replica_n22",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node34":{
          "core":"awhollynewcollection_0_shard4_replica_n23",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n24",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node36":{
          "core":"awhollynewcollection_0_shard4_replica_n25",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node37":{
          "core":"awhollynewcollection_0_shard5_replica_n26",
          "base_url":"http://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node38":{
          "core":"awhollynewcollection_0_shard5_replica_n27",
          "base_url":"http://127.0.0.1:34549/solr";,
          "node_name":"127.0.0.1:34549_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard5_replica_n29",
          "base_url":"http://127.0.0.1:41277/solr";,
          "node_name":"127.0.0.1:41277_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard5_replica_n31",
          "base_url":"http://127.0.0.1:43941/solr";,
          "node_name":"127.0.0.1:43941_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"6",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([DDCF118D149869BB:95BA653912AB462E]:0)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:758)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:764)
        at 
org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:435)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
Address already in use

Stack Trace:
java.net.BindException: Address already in use
        at 
__randomizedtesting.SeedInfo.seed([DDCF118D149869BB:56E8C25C559EC23F]:0)
        at java.base/sun.nio.ch.Net.bind0(Native Method)
        at java.base/sun.nio.ch.Net.bind(Net.java:461)
        at java.base/sun.nio.ch.Net.bind(Net.java:453)
        at 
java.base/sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:227)
        at 
java.base/sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:80)
        at 
org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
        at 
org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
        at 
org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
        at 
org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at org.eclipse.jetty.server.Server.doStart(Server.java:394)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:558)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:497)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:465)
        at 
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
11 threads leaked from SUITE scope at 
org.apache.solr.cloud.api.collections.ShardSplitTest:     1) Thread[id=89170, 
name=qtp1334635992-89170, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
[email protected]/sun.nio.ch.EPoll.wait(Native Method)         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) 
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)         
at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    2) 
Thread[id=89169, name=qtp1334635992-89169, state=RUNNABLE, 
group=TGRP-ShardSplitTest]         at 
[email protected]/sun.nio.ch.EPoll.wait(Native Method)         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) 
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)         
at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    3) 
Thread[id=89171, 
name=qtp1334635992-89171-acceptor-0@209ba0e1-ServerConnector@11a693de{HTTP/1.1,[http/1.1,
 h2c]}{127.0.0.1:34797}, state=RUNNABLE, group=TGRP-ShardSplitTest]         at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)      
   at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
         at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
         at 
app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)  
       at 
app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    4) 
Thread[id=89175, name=qtp1334635992-89175, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    5) 
Thread[id=89176, name=qtp1334635992-89176, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    6) 
Thread[id=89172, name=qtp1334635992-89172, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    7) 
Thread[id=89173, name=qtp1334635992-89173, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    8) 
Thread[id=89216, name=qtp1334635992-89216, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    9) 
Thread[id=89174, name=qtp1334635992-89174, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
         at [email protected]/java.lang.Thread.run(Thread.java:834)   10) 
Thread[id=89208, name=Scheduler-914503994, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
         at [email protected]/java.lang.Thread.run(Thread.java:834)   11) 
Thread[id=89177, name=Scheduler-2055383241, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
         at [email protected]/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 11 threads leaked from 
SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 
   1) Thread[id=89170, name=qtp1334635992-89170, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   2) Thread[id=89169, name=qtp1334635992-89169, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=89171, 
name=qtp1334635992-89171-acceptor-0@209ba0e1-ServerConnector@11a693de{HTTP/1.1,[http/1.1,
 h2c]}{127.0.0.1:34797}, state=RUNNABLE, group=TGRP-ShardSplitTest]
        at [email protected]/sun.nio.ch.ServerSocketChannelImpl.accept0(Native 
Method)
        at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at 
[email protected]/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at 
app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at 
app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   4) Thread[id=89175, name=qtp1334635992-89175, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   5) Thread[id=89176, name=qtp1334635992-89176, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   6) Thread[id=89172, name=qtp1334635992-89172, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   7) Thread[id=89173, name=qtp1334635992-89173, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   8) Thread[id=89216, name=qtp1334635992-89216, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   9) Thread[id=89174, name=qtp1334635992-89174, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
  10) Thread[id=89208, name=Scheduler-914503994, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
  11) Thread[id=89177, name=Scheduler-2055383241, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
        at __randomizedtesting.SeedInfo.seed([DDCF118D149869BB]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=89170, name=qtp1334635992-89170, state=RUNNABLE, 
group=TGRP-ShardSplitTest]         at 
[email protected]/sun.nio.ch.EPoll.wait(Native Method)         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) 
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)         
at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    2) 
Thread[id=89169, name=qtp1334635992-89169, state=RUNNABLE, 
group=TGRP-ShardSplitTest]         at 
[email protected]/sun.nio.ch.EPoll.wait(Native Method)         at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
         at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) 
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)         
at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
         at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
         at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
         at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
         at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    3) 
Thread[id=89208, name=Scheduler-914503994, state=WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
         at [email protected]/java.lang.Thread.run(Thread.java:834)    4) 
Thread[id=89177, name=Scheduler-2055383241, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]         at 
[email protected]/jdk.internal.misc.Unsafe.park(Native Method)         at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
         at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
         at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
         at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
         at [email protected]/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=89170, name=qtp1334635992-89170, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   2) Thread[id=89169, name=qtp1334635992-89169, state=RUNNABLE, 
group=TGRP-ShardSplitTest]
        at [email protected]/sun.nio.ch.EPoll.wait(Native Method)
        at 
[email protected]/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at 
[email protected]/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at 
[email protected]/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at 
app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at 
app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at 
app//org.eclipse.jetty.io.ManagedSelector$$Lambda$186/0x000000010041f440.run(Unknown
 Source)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at 
app//org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=89208, name=Scheduler-914503994, state=WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
   4) Thread[id=89177, name=Scheduler-2055383241, state=TIMED_WAITING, 
group=TGRP-ShardSplitTest]
        at [email protected]/jdk.internal.misc.Unsafe.park(Native Method)
        at 
[email protected]/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at 
[email protected]/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at 
[email protected]/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at 
[email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at [email protected]/java.lang.Thread.run(Thread.java:834)
        at __randomizedtesting.SeedInfo.seed([DDCF118D149869BB]:0)


FAILED:  org.apache.solr.client.solrj.TestLBHttp2SolrClient.testReliability

Error Message:
Timeout occured while waiting response from server at: 
http://127.0.0.1:43656/solr/collection1/select?q=*%3A*&wt=javabin&version=2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: 
http://127.0.0.1:43656/solr/collection1/select?q=*%3A*&wt=javabin&version=2
        at 
__randomizedtesting.SeedInfo.seed([19B760815EEA736C:D87FBDC7FF8CA2C5]:0)
        at 
org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:406)
        at 
org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:739)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:605)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:581)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:987)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1002)
        at 
org.apache.solr.client.solrj.TestLBHttp2SolrClient.testReliability(TestLBHttp2SolrClient.java:223)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.util.concurrent.TimeoutException
        at 
org.eclipse.jetty.client.util.InputStreamResponseListener.get(InputStreamResponseListener.java:216)
        at 
org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:397)
        ... 45 more


FAILED:  org.apache.solr.client.solrj.TestLBHttpSolrClient.testReliability

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available 
to handle this request
        at 
__randomizedtesting.SeedInfo.seed([19B760815EEA736C:D87FBDC7FF8CA2C5]:0)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:669)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:581)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:987)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1002)
        at 
org.apache.solr.client.solrj.TestLBHttpSolrClient.testReliability(TestLBHttpSolrClient.java:221)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred 
while waiting response from server at: http://127.0.0.1:37039/solr/collection1
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:605)
        ... 43 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.base/java.net.SocketInputStream.socketRead0(Native Method)
        at 
java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
        at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
        at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
        ... 46 more




Build Log:
[...truncated 15321 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest_DDCF118D149869BB-001/init-core-data-001
   [junit4]   2> 6907124 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=26 numCloses=26
   [junit4]   2> 6907124 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 6907125 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, 
clientAuth=0.0/0.0)
   [junit4]   2> 6907125 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 6907192 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 6907203 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 6907205 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 11.0.1+13-LTS
   [junit4]   2> 6907212 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6907212 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6907212 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 6907212 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@20bc1e7d{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 6907428 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@68d63bf1{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-lucene2-us-west.apache.org-40551-hdfs-_-any-16505726161215513931.dir/webapp/,AVAILABLE}{/hdfs}
   [junit4]   2> 6907429 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.AbstractConnector Started 
ServerConnector@1470d63e{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:40551}
   [junit4]   2> 6907429 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.Server Started @6907507ms
   [junit4]   2> 6907507 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 6907508 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 11.0.1+13-LTS
   [junit4]   2> 6907508 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 6907508 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 6907508 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 6907509 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@451e12a5{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 6907641 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@18600e06{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-35559-datanode-_-any-7858619906349292022.dir/webapp/,AVAILABLE}{/datanode}
   [junit4]   2> 6907641 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.AbstractConnector Started 
ServerConnector@4bb40efa{HTTP/1.1,[http/1.1]}{localhost:35559}
   [junit4]   2> 6907641 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[DDCF118D149869BB]-worker) [    
] o.e.j.s.Server Started @6907720ms
   [junit4]   2> 6907932 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xbc8d7bc0b92bd900: Processing first storage report for 
DS-f0058500-31c7-4983-942e-30e1f146d8ce from datanode 
0779ed4e-de47-4652-8f75-18a9bb121f15
   [junit4]   2> 6907932 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xbc8d7bc0b92bd900: from storage 
DS-f0058500-31c7-4983-942e-30e1f146d8ce node 
DatanodeRegistration(127.0.0.1:39844, 
datanodeUuid=0779ed4e-de47-4652-8f75-18a9bb121f15, infoPort=35554, 
infoSecurePort=0, ipcPort=38058, 
storageInfo=lv=-57;cid=testClusterID;nsid=2007588166;c=1555681552460), blocks: 
0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 6907932 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xbc8d7bc0b92bd900: Processing first storage report for 
DS-e4760d1c-5df0-471f-bda9-9687319741f8 from datanode 
0779ed4e-de47-4652-8f75-18a9bb121f15
   [junit4]   2> 6907932 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xbc8d7bc0b92bd900: from storage 
DS-e4760d1c-5df0-471f-bda9-9687319741f8 node 
DatanodeRegistration(127.0.0.1:39844, 
datanodeUuid=0779ed4e-de47-4652-8f75-18a9bb121f15, infoPort=35554, 
infoSecurePort=0, ipcPort=38058, 
storageInfo=lv=-57;cid=testClusterID;nsid=2007588166;c=1555681552460), blocks: 
0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 6908172 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testCoresAreDistributedAcrossNodes
   [junit4]   2> 6908173 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest_DDCF118D149869BB-001/tempDir-002
   [junit4]   2> 6908173 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 6908174 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6908174 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 6908274 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.c.ZkTestServer start zk server on port:43949
   [junit4]   2> 6908274 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:43949
   [junit4]   2> 6908274 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[DDCF118D149869BB])
 [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 43949
   [junit4]   2> 6908288 INFO  (zkConnectionManagerCallback-71547-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6908306 INFO  (zkConnectionManagerCallback-71549-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6908367 INFO  (zkConnectionManagerCallback-71551-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 6908379 WARN  (jetty-launcher-71552-thread-1) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 6908379 INFO  (jetty-launcher-71552-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 6908379 INFO  (jetty-launcher-71552-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]

[...truncated too long message...]

check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/null1029280419
     [copy] Copying 38 files to 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/null1029280419
   [delete] Deleting directory 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/null1029280419

resolve-example:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve-server:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null881573944
     [copy] Copying 239 files to 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null881573944
   [delete] Deleting directory 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/null881573944

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: 
org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath]         confs: [default]
[ivy:cachepath]         found 
org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath]         found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath]         found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath]         found org.apache.httpcomponents#httpclient;4.3.6 in 
public
[ivy:cachepath]         found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath]         found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath]         found commons-codec#commons-codec;1.6 in public
[ivy:cachepath]         found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 36ms :: artifacts dl 4ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for 
further details.
[wc-checker] Checking working copy status...

BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:651:
 The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:506:
 The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:493:
 Source checkout is dirty (unversioned/missing files) after running tests!!! 
Offending files:
* solr/licenses/gmetric4j-1.0.7.jar.sha1

Total time: 672 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to