Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/172/

2 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; 
[core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:56864/ux_/sa","node_name":"127.0.0.1:56864_ux_%2Fsa","state":"active","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//clusterstate.json/35)={   
"replicationFactor":"3",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node1":{           "state":"down",           
"base_url":"http://127.0.0.1:34442/ux_/sa";,           
"core":"c8n_1x3_lf_shard1_replica1",           
"node_name":"127.0.0.1:34442_ux_%2Fsa"},         "core_node2":{           
"core":"c8n_1x3_lf_shard1_replica3",           
"base_url":"http://127.0.0.1:56864/ux_/sa";,           
"node_name":"127.0.0.1:56864_ux_%2Fsa",           "state":"active",           
"leader":"true"},         "core_node3":{           
"core":"c8n_1x3_lf_shard1_replica2",           
"base_url":"http://127.0.0.1:34011/ux_/sa";,           
"node_name":"127.0.0.1:34011_ux_%2Fsa",           "state":"down"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"1",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 
1; 
[core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:56864/ux_/sa","node_name":"127.0.0.1:56864_ux_%2Fsa","state":"active","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//clusterstate.json/35)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:34442/ux_/sa";,
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:34442_ux_%2Fsa"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:56864/ux_/sa";,
          "node_name":"127.0.0.1:56864_ux_%2Fsa",
          "state":"active",
          "leader":"true"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:34011/ux_/sa";,
          "node_name":"127.0.0.1:34011_ux_%2Fsa",
          "state":"down"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([C6C1B3A24134E6CE:4E958C78EFC88B36]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:168)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:55)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=2354, name=collection2, 
state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught 
exception in thread: Thread[id=2354, name=collection2, state=RUNNABLE, 
group=TGRP-HdfsCollectionsAPIDistributedZkTest]
Caused by: java.lang.RuntimeException: 
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: https://127.0.0.1:47425/u_mx
        at __randomizedtesting.SeedInfo.seed([C6C1B3A24134E6CE]:0)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:997)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured 
while waiting response from server at: https://127.0.0.1:47425/u_mx
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1292)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1062)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1004)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:987)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:170)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
        at sun.security.ssl.InputRecord.read(InputRecord.java:503)
        at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
        at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
        at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
        at 
org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
        at 
org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
        at 
org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
        at 
org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
        at 
org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
        at 
org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
        ... 11 more




Build Log:
[...truncated 11371 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/init-core-data-001
   [junit4]   2> 106887 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 106888 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: 
/u_mx/
   [junit4]   2> 107537 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 108232 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 109023 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 109044 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 109115 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 109168 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_35640_hdfs____atzcxm/webapp
   [junit4]   2> 109885 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35640
   [junit4]   2> 110554 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 110559 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 110568 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_50811_datanode____8o6xj8/webapp
   [junit4]   2> 110923 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:50811
   [junit4]   2> 111400 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 111401 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 111417 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_52866_datanode____.390nku/webapp
   [junit4]   2> 111812 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C6C1B3A24134E6CE]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52866
   [junit4]   2> 112560 INFO  (IPC Server handler 3 on 54579) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-c0e1b2e3-9f5c-4d49-8d3b-51e4afcbbe81 node 
DatanodeRegistration(127.0.0.1:58534, 
datanodeUuid=63aaea09-6902-4d9a-be26-23f921f0f326, infoPort=59112, 
infoSecurePort=0, ipcPort=51001, 
storageInfo=lv=-56;cid=testClusterID;nsid=1904079667;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 112560 INFO  (IPC Server handler 5 on 54579) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-e350fed0-4e19-4d97-a217-ab431207def1 node 
DatanodeRegistration(127.0.0.1:37842, 
datanodeUuid=e0715786-75fc-4431-8649-d2375084280c, infoPort=36387, 
infoSecurePort=0, ipcPort=60440, 
storageInfo=lv=-56;cid=testClusterID;nsid=1904079667;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 2 msecs
   [junit4]   2> 112564 INFO  (IPC Server handler 3 on 54579) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-cc5ca218-3ef1-4efa-b6c3-2d6e78987bcd node 
DatanodeRegistration(127.0.0.1:58534, 
datanodeUuid=63aaea09-6902-4d9a-be26-23f921f0f326, infoPort=59112, 
infoSecurePort=0, ipcPort=51001, 
storageInfo=lv=-56;cid=testClusterID;nsid=1904079667;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 112564 INFO  (IPC Server handler 5 on 54579) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-5379cadf-d0ca-4a2c-a1a5-9ed93b47a5ee node 
DatanodeRegistration(127.0.0.1:37842, 
datanodeUuid=e0715786-75fc-4431-8649-d2375084280c, infoPort=36387, 
infoSecurePort=0, ipcPort=60440, 
storageInfo=lv=-56;cid=testClusterID;nsid=1904079667;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 113449 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 113460 INFO  (Thread-435) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 113460 INFO  (Thread-435) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 113560 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkTestServer start zk server on port:34805
   [junit4]   2> 113640 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 113642 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 113648 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 113649 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 113650 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 113651 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 113652 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 113653 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 113654 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 113654 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 113655 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 113904 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/control-001/cores/collection1
   [junit4]   2> 113907 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 114023 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@4c5a209f{/u_mx,null,AVAILABLE}
   [junit4]   2> 114025 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.ServerConnector Started ServerConnector@6f6e1a8a{SSL,[ssl, 
http/1.1]}{127.0.0.1:42981}
   [junit4]   2> 114025 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server Started @116218ms
   [junit4]   2> 114025 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=hdfs://localhost:54579/hdfs__localhost_54579__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001_tempDir-002_control_data,
 hostContext=/u_mx, hostPort=42981, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/control-001/cores}
   [junit4]   2> 114026 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 114026 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 114026 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 114026 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-10-10T21:52:39.186Z
   [junit4]   2> 114059 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 114059 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/control-001/solr.xml
   [junit4]   2> 114066 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 114083 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34805/solr
   [junit4]   2> 114341 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:42981_u_mx
   [junit4]   2> 114343 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.Overseer Overseer 
(id=96740072399044613-127.0.0.1:42981_u_mx-n_0000000000) starting
   [junit4]   2> 114393 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:42981_u_mx
   [junit4]   2> 114396 INFO  
(zkCallback-132-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 114411 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/control-001/cores
   [junit4]   2> 114421 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.CorePropertiesLocator Cores are: 
[collection1]
   [junit4]   2> 114517 INFO  
(OverseerStateUpdate-96740072399044613-127.0.0.1:42981_u_mx-n_0000000000) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.o.ReplicaMutator Assigning new node to 
shard shard=shard1
   [junit4]   2> 115542 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection   x:collection1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 115588 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection   x:collection1] 
o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 115805 WARN  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection   x:collection1] 
o.a.s.s.IndexSchema [collection1] default search field in schema is text. 
WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 115807 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection   x:collection1] 
o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 115818 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection   x:collection1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from 
collection control_collection
   [junit4]   2> 115827 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.HdfsDirectoryFactory 
solr.hdfs.home=hdfs://localhost:54579/solr_hdfs_home
   [junit4]   2> 115827 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication 
disabled
   [junit4]   2> 115827 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/control-001/cores/collection1],
 
dataDir=[hdfs://localhost:54579/solr_hdfs_home/control_collection/core_node1/data/]
   [junit4]   2> 115827 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr 
mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6300f876
   [junit4]   2> 115828 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
   [junit4]   2> 115912 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/control_collection/core_node1/data
   [junit4]   2> 115975 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/control_collection/core_node1/data/index
   [junit4]   2> 116014 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 116395 INFO  (IPC Server handler 1 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58534 is 
added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-5379cadf-d0ca-4a2c-a1a5-9ed93b47a5ee:NORMAL:127.0.0.1:37842|RBW],
 
ReplicaUC[[DISK]DS-cc5ca218-3ef1-4efa-b6c3-2d6e78987bcd:NORMAL:127.0.0.1:58534|RBW]]}
 size 71
   [junit4]   2> 116396 INFO  (IPC Server handler 2 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37842 is 
added to blk_1073741825_1001 size 71
   [junit4]   2> 116824 WARN  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler 
{type = requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 116856 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 116856 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= 
defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 
numVersionBuckets=65536
   [junit4]   2> 116856 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: 
tlogDfsReplication=2
   [junit4]   2> 116880 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 116880 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 116894 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.884765625, noCFSRatio=1.0]
   [junit4]   2> 117141 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.s.SolrIndexSearcher Opening 
[Searcher@622bfb1f[collection1] main]
   [junit4]   2> 117143 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO 
with znodeBase: /configs/conf1
   [junit4]   2> 117143 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path 
_rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 117145 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 117149 INFO  
(searcherExecutor-389-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore 
[collection1] Registered new searcher Searcher@622bfb1f[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 117149 INFO  
(coreLoadExecutor-388-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx c:control_collection s:shard1 r:core_node1 
x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent 
updates, using new clock 1547841162244521984
   [junit4]   2> 117175 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 117175 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 117175 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
Sync replicas to https://127.0.0.1:42981/u_mx/collection1/
   [junit4]   2> 117175 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 117175 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
https://127.0.0.1:42981/u_mx/collection1/ has no replicas
   [junit4]   2> 117179 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:42981/u_mx/collection1/ shard1
   [junit4]   2> 117330 INFO  
(coreZkRegister-381-thread-1-processing-n:127.0.0.1:42981_u_mx x:collection1 
s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:42981_u_mx 
c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController 
I am the leader, no recovery necessary
   [junit4]   2> 117577 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 117581 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection 
loss:false
   [junit4]   2> 117675 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001/cores/collection1
   [junit4]   2> 117677 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001
   [junit4]   2> 117683 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 117698 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@27933a5a{/u_mx,null,AVAILABLE}
   [junit4]   2> 117699 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.ServerConnector Started ServerConnector@3d46a3d2{SSL,[ssl, 
http/1.1]}{127.0.0.1:60878}
   [junit4]   2> 117699 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server Started @119892ms
   [junit4]   2> 117699 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=hdfs://localhost:54579/hdfs__localhost_54579__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001_tempDir-002_jetty1,
 solrconfig=solrconfig.xml, hostContext=/u_mx, hostPort=60878, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001/cores}
   [junit4]   2> 117700 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 117700 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 117700 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 117700 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-10-10T21:52:42.860Z
   [junit4]   2> 117729 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 117729 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001/solr.xml
   [junit4]   2> 117735 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 117736 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34805/solr
   [junit4]   2> 117779 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 117796 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:60878_u_mx
   [junit4]   2> 117797 INFO  
(zkCallback-132-thread-3-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 117804 INFO  
(zkCallback-141-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 117813 INFO  (zkCallback-136-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 117836 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001/cores
   [junit4]   2> 117836 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.CorePropertiesLocator Cores are: 
[collection1]
   [junit4]   2> 117839 INFO  
(OverseerStateUpdate-96740072399044613-127.0.0.1:42981_u_mx-n_0000000000) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.o.ReplicaMutator Assigning new node to 
shard shard=shard2
   [junit4]   2> 118883 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.3.0
   [junit4]   2> 118894 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 119024 WARN  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 119038 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 119051 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 119052 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.HdfsDirectoryFactory 
solr.hdfs.home=hdfs://localhost:54579/solr_hdfs_home
   [junit4]   2> 119052 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 119052 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-1-001/cores/collection1],
 dataDir=[hdfs://localhost:54579/solr_hdfs_home/collection1/core_node1/data/]
   [junit4]   2> 119052 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6300f876
   [junit4]   2> 119053 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
   [junit4]   2> 119090 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node1/data
   [junit4]   2> 119143 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node1/data/index
   [junit4]   2> 119175 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 119266 INFO  (IPC Server handler 4 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37842 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-c0e1b2e3-9f5c-4d49-8d3b-51e4afcbbe81:NORMAL:127.0.0.1:58534|RBW],
 
ReplicaUC[[DISK]DS-e350fed0-4e19-4d97-a217-ab431207def1:NORMAL:127.0.0.1:37842|FINALIZED]]}
 size 0
   [junit4]   2> 119267 INFO  (IPC Server handler 9 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58534 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-c0e1b2e3-9f5c-4d49-8d3b-51e4afcbbe81:NORMAL:127.0.0.1:58534|RBW],
 
ReplicaUC[[DISK]DS-e350fed0-4e19-4d97-a217-ab431207def1:NORMAL:127.0.0.1:37842|FINALIZED]]}
 size 0
   [junit4]   2> 119284 WARN  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 119361 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 119361 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 119361 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 119384 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 119384 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 119387 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.884765625, noCFSRatio=1.0]
   [junit4]   2> 119419 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@433a9a54[collection1] main]
   [junit4]   2> 119420 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 119421 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 119421 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 119421 INFO  
(searcherExecutor-400-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered 
new searcher Searcher@433a9a54[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 119422 INFO  
(coreLoadExecutor-399-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1547841164627935232
   [junit4]   2> 119430 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough 
replicas found to continue.
   [junit4]   2> 119430 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may 
be the new leader - try and sync
   [junit4]   2> 119430 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:60878/u_mx/collection1/
   [junit4]   2> 119430 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now 
sync replicas to me
   [junit4]   2> 119430 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
https://127.0.0.1:60878/u_mx/collection1/ has no replicas
   [junit4]   2> 119434 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am 
the new leader: https://127.0.0.1:60878/u_mx/collection1/ shard2
   [junit4]   2> 119585 INFO  
(coreZkRegister-394-thread-1-processing-n:127.0.0.1:60878_u_mx x:collection1 
s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:60878_u_mx c:collection1 
s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no 
recovery necessary
   [junit4]   2> 119984 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001/cores/collection1
   [junit4]   2> 119985 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001
   [junit4]   2> 119989 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 119993 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1f75d63a{/u_mx,null,AVAILABLE}
   [junit4]   2> 119995 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.ServerConnector Started ServerConnector@1f42b2f5{SSL,[ssl, 
http/1.1]}{127.0.0.1:41445}
   [junit4]   2> 119995 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server Started @122188ms
   [junit4]   2> 119995 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=hdfs://localhost:54579/hdfs__localhost_54579__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001_tempDir-002_jetty2,
 solrconfig=solrconfig.xml, hostContext=/u_mx, hostPort=41445, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001/cores}
   [junit4]   2> 119996 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 119996 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 119996 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 119996 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-10-10T21:52:45.156Z
   [junit4]   2> 120030 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 120030 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001/solr.xml
   [junit4]   2> 120036 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 120037 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34805/solr
   [junit4]   2> 120066 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 120069 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:41445_u_mx
   [junit4]   2> 120069 INFO  
(zkCallback-141-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 120071 INFO  (zkCallback-136-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 120076 INFO  
(zkCallback-132-thread-1-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 120088 INFO  
(zkCallback-147-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 120100 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001/cores
   [junit4]   2> 120100 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.CorePropertiesLocator Cores are: 
[collection1]
   [junit4]   2> 120118 INFO  
(OverseerStateUpdate-96740072399044613-127.0.0.1:42981_u_mx-n_0000000000) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.o.ReplicaMutator Assigning new node to 
shard shard=shard1
   [junit4]   2> 121293 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.3.0
   [junit4]   2> 121338 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 121417 WARN  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 121429 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 121440 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 121440 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.HdfsDirectoryFactory 
solr.hdfs.home=hdfs://localhost:54579/solr_hdfs_home
   [junit4]   2> 121440 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 121440 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-2-001/cores/collection1],
 dataDir=[hdfs://localhost:54579/solr_hdfs_home/collection1/core_node2/data/]
   [junit4]   2> 121440 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6300f876
   [junit4]   2> 121441 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
   [junit4]   2> 121462 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node2/data
   [junit4]   2> 121516 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node2/data/index
   [junit4]   2> 121542 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 121594 INFO  (IPC Server handler 4 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37842 is 
added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-c0e1b2e3-9f5c-4d49-8d3b-51e4afcbbe81:NORMAL:127.0.0.1:58534|RBW],
 
ReplicaUC[[DISK]DS-5379cadf-d0ca-4a2c-a1a5-9ed93b47a5ee:NORMAL:127.0.0.1:37842|RBW]]}
 size 0
   [junit4]   2> 121594 INFO  (IPC Server handler 9 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58534 is 
added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-5379cadf-d0ca-4a2c-a1a5-9ed93b47a5ee:NORMAL:127.0.0.1:37842|RBW],
 
ReplicaUC[[DISK]DS-cc5ca218-3ef1-4efa-b6c3-2d6e78987bcd:NORMAL:127.0.0.1:58534|FINALIZED]]}
 size 0
   [junit4]   2> 121618 WARN  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 121694 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 121694 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 121694 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 121712 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 121712 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 121717 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.884765625, noCFSRatio=1.0]
   [junit4]   2> 121742 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@2f2431e3[collection1] main]
   [junit4]   2> 121743 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 121743 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 121743 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 121744 INFO  
(searcherExecutor-411-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] Registered 
new searcher Searcher@2f2431e3[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 121744 INFO  
(coreLoadExecutor-410-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1547841167062728704
   [junit4]   2> 121755 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough 
replicas found to continue.
   [junit4]   2> 121755 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may 
be the new leader - try and sync
   [junit4]   2> 121755 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:41445/u_mx/collection1/
   [junit4]   2> 121755 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now 
sync replicas to me
   [junit4]   2> 121755 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy 
https://127.0.0.1:41445/u_mx/collection1/ has no replicas
   [junit4]   2> 121769 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am 
the new leader: https://127.0.0.1:41445/u_mx/collection1/ shard1
   [junit4]   2> 121920 INFO  
(coreZkRegister-405-thread-1-processing-n:127.0.0.1:41445_u_mx x:collection1 
s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:41445_u_mx c:collection1 
s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no 
recovery necessary
   [junit4]   2> 121994 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001/cores/collection1
   [junit4]   2> 121995 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001
   [junit4]   2> 121997 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 122085 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@75fbc71e{/u_mx,null,AVAILABLE}
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.ServerConnector Started ServerConnector@4bb2fd26{SSL,[ssl, 
http/1.1]}{127.0.0.1:47425}
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server Started @124280ms
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=hdfs://localhost:54579/hdfs__localhost_54579__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001_tempDir-002_jetty3,
 solrconfig=solrconfig.xml, hostContext=/u_mx, hostPort=47425, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001/cores}
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 122087 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-10-10T21:52:47.247Z
   [junit4]   2> 122133 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 122133 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001/solr.xml
   [junit4]   2> 122235 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 122236 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34805/solr
   [junit4]   2> 122294 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:47425_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (3)
   [junit4]   2> 122299 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:47425_u_mx    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:47425_u_mx
   [junit4]   2> 122300 INFO  (zkCallback-136-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 122300 INFO  
(zkCallback-141-thread-1-processing-n:127.0.0.1:60878_u_mx) 
[n:127.0.0.1:60878_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 122301 INFO  
(zkCallback-147-thread-1-processing-n:127.0.0.1:41445_u_mx) 
[n:127.0.0.1:41445_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 122302 INFO  
(zkCallback-132-thread-2-processing-n:127.0.0.1:42981_u_mx) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 122308 INFO  
(zkCallback-153-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 122357 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:47425_u_mx    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001/cores
   [junit4]   2> 122357 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) 
[n:127.0.0.1:47425_u_mx    ] o.a.s.c.CorePropertiesLocator Cores are: 
[collection1]
   [junit4]   2> 122372 INFO  
(OverseerStateUpdate-96740072399044613-127.0.0.1:42981_u_mx-n_0000000000) 
[n:127.0.0.1:42981_u_mx    ] o.a.s.c.o.ReplicaMutator Assigning new node to 
shard shard=shard2
   [junit4]   2> 123442 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.3.0
   [junit4]   2> 123478 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 123605 WARN  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 123614 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 123630 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 123630 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.HdfsDirectoryFactory 
solr.hdfs.home=hdfs://localhost:54579/solr_hdfs_home
   [junit4]   2> 123630 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 123630 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-3-001/cores/collection1],
 dataDir=[hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data/]
   [junit4]   2> 123630 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6300f876
   [junit4]   2> 123631 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data/snapshot_metadata
   [junit4]   2> 123686 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data
   [junit4]   2> 123710 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data/index
   [junit4]   2> 123740 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 123827 INFO  (IPC Server handler 4 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58534 is 
added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-e350fed0-4e19-4d97-a217-ab431207def1:NORMAL:127.0.0.1:37842|RBW],
 
ReplicaUC[[DISK]DS-c0e1b2e3-9f5c-4d49-8d3b-51e4afcbbe81:NORMAL:127.0.0.1:58534|RBW]]}
 size 0
   [junit4]   2> 123834 INFO  (IPC Server handler 5 on 54579) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37842 is 
added to blk_1073741828_1004 size 71
   [junit4]   2> 123846 WARN  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 123903 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 123903 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 123903 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
   [junit4]   2> 123933 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 123933 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 123943 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.884765625, noCFSRatio=1.0]
   [junit4]   2> 123980 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@1c67c661[collection1] main]
   [junit4]   2> 123983 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 123983 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 123983 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 123984 INFO  
(searcherExecutor-422-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [collection1] Registered 
new searcher Searcher@1c67c661[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 123988 INFO  
(coreLoadExecutor-421-thread-1-processing-n:127.0.0.1:47425_u_mx) 
[n:127.0.0.1:47425_u_mx c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1547841169415733248
   [junit4]   2> 124004 INFO  
(coreZkRegister-416-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to 
recover:collection1
   [junit4]   2> 124017 INFO  
(updateExecutor-150-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running 
recovery
   [junit4]   2> 124017 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery 
process. recoveringAfterStartup=true
   [junit4]   2> 124018 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### 
startupVersions=[[]]
   [junit4]   2> 124018 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering 
updates. core=[collection1]
   [junit4]   2> 124018 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer 
updates. HDFSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 124018 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state 
of core [collection1] as recovering, leader is 
[https://127.0.0.1:60878/u_mx/collection1/] and I am 
[https://127.0.0.1:47425/u_mx/collection1/]
   [junit4]   2> 124026 INFO  
(recoveryExecutor-151-thread-1-processing-n:127.0.0.1:47425_u_mx x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:47425_u_mx c:collection1 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep 
recovery command to [https://127.0.0.1:60878/u_mx]; [WaitForState: 
action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:47425_u_mx&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 124102 INFO  (qtp1793007359-1177) [n:127.0.0.1:60878_u_mx    ] 
o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: 
recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 124112 INFO  (qtp1793007359-1177) [n:127.0.0.1:60878_u_mx    ] 
o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 
(shard2 of collection1) have state: recovering
   [junit4]   2> 124112 INFO  (qtp1793007359-1177) [n:127.0.0.1:60878_u_mx    ] 
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, 
shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? 
true, live=true, checkLive=true, currentState=recovering, localState=active, 
nodeName=127.0.0.1:47425_u_mx, coreNodeName=core_node3, 
onlyIfActiveCheckResult=false, nodeProps: 
core_node3:{"core":"collection1","dataDir":"hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data/","base_url":"https://127.0.0.1:47425/u_mx","node_name":"127.0.0.1:47425_u_mx","state":"recovering","ulogDir":"hdfs://localhost:54579/solr_hdfs_home/collection1/core_node3/data/tlog"}
   [junit4]   2> 124112 INFO  (qtp1793007359-1177) [n:127.0.0.1:60878_u_mx    ] 
o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, 
checkLive: true, onlyIfLeader: true for: 0 seconds.
   [junit4]   2> 124112 INFO  (qtp1793007359-1177) [n:127.0.0.1:60878_u_mx    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={nodeName=127.0.0.1:47425_u_mx&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2}
 status=0 QTime=10
   [junit4]   2> 124648 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-4-001/cores/collection1
   [junit4]   2> 124651 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C6C1B3A24134E6CE-001/shard-4-001
   [junit4]   2> 124663 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C6C1B3A24134E6CE]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [j

[...truncated too long message...]

cheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-6.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar
 com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20161010_215042_928.events
 
@/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J2-20161010_215042_928.suites
 -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from 
the forked process? This may indicate JVM crash or runner bugs.
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4]     at 
com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4]     at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]     at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]     at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]     at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:773:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:709:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:59:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build.xml:252:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/common-build.xml:536:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:1443:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:984:
 At least one slave process threw an exception, first: Quit event not received 
from the forked process? This may indicate JVM crash or runner bugs.

Total time: 257 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to