Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/565/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
  at sun.reflect.GeneratedConstructorAccessor177.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)  at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)  at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
        at sun.reflect.GeneratedConstructorAccessor177.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
        at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([D8E0DA520323B775]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
        at sun.reflect.GeneratedMethodAccessor50.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12365 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/init-core-data-001
   [junit4]   2> 2464645 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2464647 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/tempDir-001
   [junit4]   2> 2464647 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2464647 INFO  (Thread-9496) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2464647 INFO  (Thread-9496) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 2464752 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:62089
   [junit4]   2> 2464765 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2464768 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@525a4a06{/solr,null,AVAILABLE}
   [junit4]   2> 2464769 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2464770 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@b55de9e{/solr,null,AVAILABLE}
   [junit4]   2> 2464770 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@5db77299{SSL,[ssl, 
http/1.1]}{127.0.0.1:65194}
   [junit4]   2> 2464770 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.e.j.s.Server Started @2469813ms
   [junit4]   2> 2464770 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=65194}
   [junit4]   2> 2464771 ERROR (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2464771 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 2464771 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2464771 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2464771 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-21T17:23:10.708Z
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@145abed7{SSL,[ssl, 
http/1.1]}{127.0.0.1:55719}
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.e.j.s.Server Started @2469814ms
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=55719}
   [junit4]   2> 2464772 ERROR (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2464772 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-21T17:23:10.709Z
   [junit4]   2> 2464779 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2464779 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2464800 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2464800 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2464801 INFO  (jetty-launcher-10428-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62089/solr
   [junit4]   2> 2464802 INFO  (jetty-launcher-10428-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62089/solr
   [junit4]   2> 2464839 INFO  (jetty-launcher-10428-thread-2) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:65194_solr
   [junit4]   2> 2464840 INFO  (jetty-launcher-10428-thread-1) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:55719_solr
   [junit4]   2> 2464840 INFO  (jetty-launcher-10428-thread-2) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.Overseer Overseer 
(id=97146699159044102-127.0.0.1:65194_solr-n_0000000000) starting
   [junit4]   2> 2464842 INFO  
(zkCallback-10439-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2464842 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2464856 INFO  (jetty-launcher-10428-thread-2) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:65194_solr
   [junit4]   2> 2464876 INFO  
(zkCallback-10439-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2464876 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2464914 INFO  (jetty-launcher-10428-thread-2) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/tempDir-001/node2/.
   [junit4]   2> 2464946 INFO  (jetty-launcher-10428-thread-1) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/tempDir-001/node1/.
   [junit4]   2> 2464954 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2464957 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:62089/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 2465052 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 2465073 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2465074 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2465097 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_solaris.vm_59123_hdfs____1hz9nk/webapp
   [junit4]   2> 2465629 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:59123
   [junit4]   2> 2466605 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2466606 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2466619 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_45415_datanode____.x02ixu/webapp
   [junit4]   2> 2467175 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45415
   [junit4]   2> 2467303 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2467304 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 2467331 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_61686_datanode____w0o5mo/webapp
   [junit4]   2> 2467339 INFO  (IPC Server handler 4 on 59551) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-aebe163d-7a4a-4587-bf13-201f6e06873f node 
DatanodeRegistration(127.0.0.1:57821, 
datanodeUuid=f9daf6c3-7bbe-43d4-9609-1b8016f90ff9, infoPort=37906, 
infoSecurePort=0, ipcPort=47062, 
storageInfo=lv=-56;cid=testClusterID;nsid=1508917533;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2467339 INFO  (IPC Server handler 4 on 59551) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-984eb835-a336-4a07-a42d-aae3183744c9 node 
DatanodeRegistration(127.0.0.1:57821, 
datanodeUuid=f9daf6c3-7bbe-43d4-9609-1b8016f90ff9, infoPort=37906, 
infoSecurePort=0, ipcPort=47062, 
storageInfo=lv=-56;cid=testClusterID;nsid=1508917533;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2467844 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:61686
   [junit4]   2> 2467962 INFO  (IPC Server handler 5 on 59551) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-433cf706-2c30-4730-8bdc-8eb58d8b8449 node 
DatanodeRegistration(127.0.0.1:56429, 
datanodeUuid=c78570b9-6fd0-4fb5-9b40-f51208a81c0d, infoPort=39895, 
infoSecurePort=0, ipcPort=45627, 
storageInfo=lv=-56;cid=testClusterID;nsid=1508917533;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 2467963 INFO  (IPC Server handler 5 on 59551) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-479991bc-4b18-4940-a9b0-d6fc8cfc803d node 
DatanodeRegistration(127.0.0.1:56429, 
datanodeUuid=c78570b9-6fd0-4fb5-9b40-f51208a81c0d, infoPort=39895, 
infoSecurePort=0, ipcPort=45627, 
storageInfo=lv=-56;cid=testClusterID;nsid=1508917533;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2468049 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D8E0DA520323B775]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 2468067 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2468071 INFO  
(OverseerThreadFactory-11332-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 2468184 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2468199 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2468309 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2468309 INFO  
(zkCallback-10439-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2469200 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.0
   [junit4]   2> 2469211 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.0
   [junit4]   2> 2469214 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica1] Schema name=minimal
   [junit4]   2> 2469220 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2469222 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica1' using configuration from 
collection recoverytest
   [junit4]   2> 2469224 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica2] Schema name=minimal
   [junit4]   2> 2469241 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2469242 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica2' using configuration from 
collection recoverytest
   [junit4]   2> 2469252 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:59551/data
   [junit4]   2> 2469253 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2469253 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica2] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/tempDir-001/node1/recoverytest_shard1_replica2],
 dataDir=[hdfs://solaris-vm:59551/data/recoverytest/core_node2/data/]
   [junit4]   2> 2469253 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 2469253 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:59551/data
   [junit4]   2> 2469253 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2469253 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica1] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D8E0DA520323B775-001/tempDir-001/node2/recoverytest_shard1_replica1],
 dataDir=[hdfs://solaris-vm:59551/data/recoverytest/core_node1/data/]
   [junit4]   2> 2469254 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 2469266 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2469266 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2469266 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2469266 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2469285 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2469287 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node1/data
   [junit4]   2> 2469289 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2469290 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node2/data
   [junit4]   2> 2469330 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node2/data/index
   [junit4]   2> 2469334 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://solaris-vm:59551/data/recoverytest/core_node1/data/index
   [junit4]   2> 2469342 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2469342 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2469349 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2469351 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 2469351 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 2469400 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 2469402 INFO  (IPC Server handler 4 on 59551) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56429 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-984eb835-a336-4a07-a42d-aae3183744c9:NORMAL:127.0.0.1:57821|RBW],
 
ReplicaUC[[DISK]DS-479991bc-4b18-4940-a9b0-d6fc8cfc803d:NORMAL:127.0.0.1:56429|RBW]]}
 size 0
   [junit4]   2> 2469403 INFO  (IPC Server handler 6 on 59551) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57821 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-479991bc-4b18-4940-a9b0-d6fc8cfc803d:NORMAL:127.0.0.1:56429|RBW],
 
ReplicaUC[[DISK]DS-aebe163d-7a4a-4587-bf13-201f6e06873f:NORMAL:127.0.0.1:57821|FINALIZED]]}
 size 0
   [junit4]   2> 2469427 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2469427 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2469427 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2469441 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2469441 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2469460 INFO  (IPC Server handler 8 on 59551) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56429 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-984eb835-a336-4a07-a42d-aae3183744c9:NORMAL:127.0.0.1:57821|RBW],
 
ReplicaUC[[DISK]DS-433cf706-2c30-4730-8bdc-8eb58d8b8449:NORMAL:127.0.0.1:56429|RBW]]}
 size 0
   [junit4]   2> 2469461 INFO  (IPC Server handler 7 on 59551) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57821 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-984eb835-a336-4a07-a42d-aae3183744c9:NORMAL:127.0.0.1:57821|RBW],
 
ReplicaUC[[DISK]DS-433cf706-2c30-4730-8bdc-8eb58d8b8449:NORMAL:127.0.0.1:56429|RBW]]}
 size 0
   [junit4]   2> 2469489 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2469489 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2469489 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2469502 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@4133ffc0[recoverytest_shard1_replica2] main]
   [junit4]   2> 2469504 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2469505 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 2469505 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 2469506 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2469506 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2469507 INFO  
(searcherExecutor-11338-thread-1-processing-n:127.0.0.1:55719_solr 
x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica2] Registered new searcher 
Searcher@4133ffc0[recoverytest_shard1_replica2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2469508 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1554347191639736320
   [junit4]   2> 2469523 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 2469524 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@20c5bf7a[recoverytest_shard1_replica1] main]
   [junit4]   2> 2469527 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2469528 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2469528 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2469528 INFO  
(searcherExecutor-11337-thread-1-processing-n:127.0.0.1:65194_solr 
x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica1] Registered new searcher 
Searcher@20c5bf7a[recoverytest_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2469528 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1554347191660707840
   [junit4]   2> 2469627 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2469627 INFO  
(zkCallback-10439-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470025 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2470025 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2470025 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Sync replicas to https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/
   [junit4]   2> 2470025 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=https://127.0.0.1:55719/solr 
START replicas=[https://127.0.0.1:65194/solr/recoverytest_shard1_replica1/] 
nUpdates=100
   [junit4]   2> 2470025 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=https://127.0.0.1:55719/solr 
DONE.  We have no versions.  sync failed.
   [junit4]   2> 2470032 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.c.S.Request [recoverytest_shard1_replica1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 2470033 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2470033 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2470039 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/ shard1
   [junit4]   2> 2470145 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470145 INFO  
(zkCallback-10439-thread-3-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470192 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 2470196 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=1998
   [junit4]   2> 2470299 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470299 INFO  
(zkCallback-10439-thread-3-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470546 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=2363
   [junit4]   2> 2470551 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 2470651 INFO  
(zkCallback-10438-thread-1-processing-n:127.0.0.1:55719_solr) 
[n:127.0.0.1:55719_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2470651 INFO  
(zkCallback-10439-thread-1-processing-n:127.0.0.1:65194_solr) 
[n:127.0.0.1:65194_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 2471552 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=3486
   [junit4]   2> 2471554 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D8E0DA520323B775]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 1350 documents
   [junit4]   2> 2471613 INFO  (qtp1854111983-50441) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0
 (1554347193802948609)]} 0 25
   [junit4]   2> 2471614 INFO  (qtp1265281529-50433) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1554347193802948609)]} 0 43
   [junit4]   2> 2471616 INFO  (qtp1854111983-50450) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0
 (1554347193802948608)]} 0 30
   [junit4]   2> 2471617 INFO  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1554347193802948608)]} 0 46
   [junit4]   2> 2471622 INFO  (qtp1854111983-50440) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1
 (1554347193851183104)]} 0 2
   [junit4]   2> 2471623 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1554347193851183104)]} 0 6
   [junit4]   2> 2471625 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1
 (1554347193854328832)]} 0 1
   [junit4]   2> 2471625 INFO  (qtp1265281529-50429) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1554347193854328832)]} 0 5
   [junit4]   2> 2471631 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-1
 (-1554347193860620288)]} 0 1
   [junit4]   2> 2471631 INFO  (qtp1265281529-50427) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 
(-1554347193860620288)]} 0 5
   [junit4]   2> 2471633 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-1
 (-1554347193863766016)]} 0 1
   [junit4]   2> 2471633 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 
(-1554347193863766016)]} 0 4
   [junit4]   2> 2471638 INFO  (qtp1854111983-50452) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2
 (1554347193869008896)]} 0 1
   [junit4]   2> 2471638 INFO  (qtp1265281529-50430) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1554347193869008896)]} 0 4
   [junit4]   2> 2471639 INFO  (qtp1854111983-50436) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2
 (1554347193871106048)]} 0 1
   [junit4]   2> 2471639 INFO  (qtp1265281529-50433) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1554347193871106048)]} 0 3
   [junit4]   2> 2471646 INFO  (qtp1854111983-50450) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3
 (1554347193877397504)]} 0 1
   [junit4]   2> 2471646 INFO  (qtp1854111983-50441) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3
 (1554347193876348928)]} 0 1
   [junit4]   2> 2471646 INFO  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1554347193876348928)]} 0 5
   [junit4]   2> 2471647 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 
(1554347193877397504)]} 0 4
   [junit4]   2> 2471656 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4
 (1554347193884737536)]} 0 1
   [junit4]   2> 2471656 INFO  (qtp1854111983-50440) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4
 (1554347193884737537)]} 0 1
   [junit4]   2> 2471657 INFO  (qtp1265281529-50429) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 
(1554347193884737536)]} 0 7
   [junit4]   2> 2471657 INFO  (qtp1265281529-50427) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 
(1554347193884737537)]} 0 7
   [junit4]   2> 2471668 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-4
 (-1554347193900466177)]} 0 1
   [junit4]   2> 2471668 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-4
 (-1554347193900466176)]} 0 2
   [junit4]   2> 2471669 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 
(-1554347193900466176)]} 0 5
   [junit4]   2> 2471671 INFO  (qtp1265281529-50430) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 
(-1554347193900466177)]} 0 7
   [junit4]   2> 2471682 INFO  (qtp1854111983-50452) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5
 (1554347193909903360)]} 0 3
   [junit4]   2> 2471683 INFO  (qtp1854111983-50436) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5
 (1554347193915146240)]} 0 1
   [junit4]   2> 2471684 INFO  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 
(1554347193915146240)]} 0 6
   [junit4]   2> 2471685 INFO  (qtp1265281529-50433) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 
(1554347193909903360)]} 0 12
   [junit4]   2> 2471700 INFO  (qtp1854111983-50452) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6
 (1554347193928777728)]} 0 3
   [junit4]   2> 2471701 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 
(1554347193928777728)]} 0 10
   [junit4]   2> 2471701 INFO  (qtp1854111983-50441) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6
 (1554347193935069184)]} 0 2
   [junit4]   2> 2471702 INFO  (qtp1265281529-50429) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 
(1554347193935069184)]} 0 4
   [junit4]   2> 2471709 INFO  (qtp1854111983-50440) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6
 (-1554347193943457793)]} 0 1
   [junit4]   2> 2471709 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6
 (-1554347193943457792)]} 0 1
   [junit4]   2> 2471710 INFO  (qtp1265281529-50427) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 
(-1554347193943457793)]} 0 4
   [junit4]   2> 2471710 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 
(-1554347193943457792)]} 0 4
   [junit4]   2> 2471717 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7
 (1554347193951846400)]} 0 1
   [junit4]   2> 2471717 INFO  (qtp1854111983-50450) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7
 (1554347193952894976)]} 0 1
   [junit4]   2> 2471717 INFO  (qtp1265281529-50430) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 
(1554347193951846400)]} 0 4
   [junit4]   2> 2471718 INFO  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 
(1554347193952894976)]} 0 4
   [junit4]   2> 2471723 INFO  (qtp1854111983-50436) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8
 (1554347193959186433)]} 0 1
   [junit4]   2> 2471724 INFO  (qtp1854111983-50452) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8
 (1554347193959186432)]} 0 1
   [junit4]   2> 2471725 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 
(1554347193959186433)]} 0 4
   [junit4]   2> 2471725 INFO  (qtp1265281529-50433) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 
(1554347193959186432)]} 0 5
   [junit4]   2> 2471731 INFO  (qtp1854111983-50441) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9
 (1554347193967575040)]} 0 1
   [junit4]   2> 2471731 INFO  (qtp1854111983-50440) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9
 (1554347193967575041)]} 0 1
   [junit4]   2> 2471731 INFO  (qtp1265281529-50429) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 
(1554347193967575041)]} 0 3
   [junit4]   2> 2471732 INFO  (qtp1265281529-50427) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 
(1554347193967575040)]} 0 4
   [junit4]   2> 2471738 INFO  (qtp1854111983-50450) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-8
 (-1554347193973866496)]} 0 1
   [junit4]   2> 2471738 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8
 (-1554347193973866497)]} 0 1
   [junit4]   2> 2471738 INFO  (qtp1265281529-50430) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 
(-1554347193973866496)]} 0 4
   [junit4]   2> 2471739 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 
(-1554347193973866497)]} 0 4
   [junit4]   2> 2471743 INFO  (qtp1854111983-50452) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10
 (1554347193981206528)]} 0 0
   [junit4]   2> 2471743 INFO  (qtp1854111983-50443) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10
 (1554347193981206529)]} 0 0
   [junit4]   2> 2471744 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 
(1554347193981206529)]} 0 3
   [junit4]   2> 2471744 INFO  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 
(1554347193981206528)]} 0 3
   [junit4]   2> 2471749 INFO  (qtp1854111983-50439) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-10
 (-1554347193986449408)]} 0 1
   [junit4]   2> 2471749 INFO  (qtp1854111983-50436) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-10
 (-1554347193986449409)]} 0 1
   [junit4]   2> 2471749 INFO  (qtp1265281529-50429) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 
(-1554347193986449409)]} 0 3
   [junit4]   2> 2471750 INFO  (qtp1265281529-50433) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 
(-1554347193986449408)]} 0 3
   [junit4]   2> 2471755 INFO  (qtp1854111983-50442) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11
 (1554347193992740864)]} 0 1
   [junit4]   2> 2471755 INFO  (qtp1854111983-50441) [n:127.0.0.1:65194_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:55719/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11
 (1554347193992740865)]} 0 1
   [junit4]   2> 2471755 INFO  (qtp1265281529-50427) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 
(1554347193992740864)]} 0 3
   [junit4]   2> 2471755 INFO  (qtp1265281529-50430) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 
(1554347193992740865)]} 0 3
   [junit4]   2> 2471758 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D8E0DA520323B775]) [    ] 
o.e.j.s.AbstractConnector Stopped ServerConnector@5db77299{SSL,[ssl, 
http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2471760 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D8E0DA520323B775]) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=865680354
   [junit4]   2> 2471761 ERROR 
(updateExecutor-10432-thread-1-processing-https:////127.0.0.1:65194//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:55719_solr s:shard1 
c:recoverytest) [n:127.0.0.1:55719_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471761 WARN  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
https://127.0.0.1:65194/solr
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471762 ERROR 
(updateExecutor-10432-thread-2-processing-https:////127.0.0.1:65194//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:55719_solr s:shard1 
c:recoverytest) [n:127.0.0.1:55719_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471762 ERROR (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica https://127.0.0.1:65194/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471763 WARN  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
https://127.0.0.1:65194/solr
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471763 ERROR (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica https://127.0.0.1:65194/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471763 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica1 
coreNodeName=core_node1 on 127.0.0.1:65194_solr into leader-initiated recovery.
   [junit4]   2> 2471763 INFO  (qtp1265281529-50428) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 
(1554347193999032321)]} 0 5
   [junit4]   2> 2471763 INFO  (qtp1265281529-50431) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 
(1554347193999032320)]} 0 5
   [junit4]   2> 2471764 INFO  (coreCloseExecutor-11347-thread-1) 
[n:127.0.0.1:65194_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] 
 CLOSING SolrCore org.apache.solr.core.SolrCore@3b34096b
   [junit4]   2> 2471767 ERROR 
(updateExecutor-10432-thread-1-processing-https:////127.0.0.1:65194//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:55719_solr s:shard1 
c:recoverytest) [n:127.0.0.1:55719_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471769 WARN  (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
https://127.0.0.1:65194/solr
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:409)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2471770 ERROR (qtp1265281529-50432) [n:127.0.0.1:55719_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica https://127.0.0.1:65194/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
   [junit4]   2>        at 
org.apache.http.conn.ssl.SSLSocketFactory.connectSocket(SSLSocketFactory.java:532)
   [junit4]   2>        at org.apache.ht

[...truncated too long message...]

erver.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 54 more
   [junit4]   2> 2485714 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:62089 62089
   [junit4]   2> 2485714 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D8E0DA520323B775]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): 
{rnd_b=PostingsFormat(name=LuceneVarGapFixedInterval), 
_version_=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), 
a_i=PostingsFormat(name=LuceneVarGapFixedInterval), 
id=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, 
maxPointsInLeafNode=2001, maxMBSortInHeap=7.326711905488869, 
sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=uk-UA, 
timezone=Antarctica/Palmer
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 
(64-bit)/cpus=3,threads=2,free=202842904,total=515899392
   [junit4]   2> NOTE: All tests run in this JVM: [CheckHdfsIndexTest, 
StressHdfsTest, DistribCursorPagingTest, BasicAuthIntegrationTest, 
TestWriterPerf, TestValueSourceCache, UpdateRequestProcessorFactoryTest, 
TestDocSet, TestQuerySenderNoQuery, TestStressLiveNodes, TestConfig, 
DeleteShardTest, DistributedIntervalFacetingTest, SolrIndexConfigTest, 
AutoCommitTest, TestAuthenticationFramework, DeleteInactiveReplicaTest, 
SpellPossibilityIteratorTest, AddBlockUpdateTest, TestManagedResourceStorage, 
TermVectorComponentTest, BlockJoinFacetDistribTest, TestFileDictionaryLookup, 
TestDFRSimilarityFactory, SimpleCollectionCreateDeleteTest, 
CoreAdminRequestStatusTest, BigEndianAscendingWordSerializerTest, 
CdcrReplicationHandlerTest, TestRecoveryHdfs, CursorMarkTest, 
TestObjectReleaseTracker, BasicZkTest, ShowFileRequestHandlerTest, 
TestRangeQuery, RemoteQueryErrorTest, TestDFISimilarityFactory, 
TestPHPSerializedResponseWriter, SpatialHeatmapFacetsTest, 
TestLMDirichletSimilarityFactory, OverseerModifyCollectionTest, TestTrie, 
ReplicationFactorTest, TestAtomicUpdateErrorCases, 
HdfsChaosMonkeyNothingIsSafeTest, TestMaxScoreQueryParser, 
SolrCloudExampleTest, TestCoreDiscovery, TestSQLHandlerNonCloud, 
OpenCloseCoreStressTest, TestFieldCacheReopen, TestNumericTerms64, 
DebugComponentTest, ChangedSchemaMergeTest, UUIDUpdateProcessorFallbackTest, 
SegmentsInfoRequestHandlerTest, TestFuzzyAnalyzedSuggestions, 
TestDeleteCollectionOnDownNodes, BasicDistributedZk2Test, 
TestLeaderElectionZkExpiry, PreAnalyzedFieldTest, TestCloudPseudoReturnFields, 
TestLocalFSCloudBackupRestore, TestAuthorizationFramework, TestCrossCoreJoin, 
DefaultValueUpdateProcessorTest, ShardRoutingCustomTest, 
TestSolrFieldCacheMBean, TestEmbeddedSolrServerConstructors, 
TestStressUserVersions, TestCustomDocTransformer, CdcrUpdateLogTest, 
TriLevelCompositeIdRoutingTest, CollectionsAPIDistributedZkTest, 
TestSolrQueryParserResource, TestSolrCLIRunExample, 
CurrencyFieldOpenExchangeTest, TestFieldCollectionResource, 
TestCopyFieldCollectionResource, CSVRequestHandlerTest, 
HdfsCollectionsAPIDistributedZkTest, TestDynamicFieldCollectionResource, 
ExplicitHLLTest, TestSortByMinMaxFunction, TestGeoJSONResponseWriter, 
DistributedSuggestComponentTest, ClusterStateTest, TestDownShardTolerantSearch, 
TestExpandComponent, RAMDirectoryFactoryTest, 
TestLMJelinekMercerSimilarityFactory, ExitableDirectoryReaderTest, 
DistributedVersionInfoTest, OverseerStatusTest, TestSSLRandomization, 
ZkStateReaderTest, RankQueryTest, VersionInfoTest, TestClusterProperties, 
TestRebalanceLeaders, HLLSerializationTest, DirectUpdateHandlerOptimizeTest, 
CreateCollectionCleanupTest, TestLuceneMatchVersion, BasicFunctionalityTest, 
AtomicUpdatesTest, ConnectionReuseTest, DateMathParserTest, 
HdfsRecoverLeaseTest, FullSolrCloudDistribCmdsTest, TestBadConfig, 
ResourceLoaderTest, TestSerializedLuceneMatchVersion, HighlighterMaxOffsetTest, 
UpdateParamsTest, TestSolrDeletionPolicy2, TestCorePropertiesReload, 
TestXmlQParser, CoreSorterTest, TestDocTermOrds, TestReplicationHandlerBackup, 
TestSolrCoreProperties, SignatureUpdateProcessorFactoryTest, 
RollingRestartTest, CircularListTest, CacheHeaderTest, 
CollectionTooManyReplicasTest, TestSolr4Spatial, SpatialRPTFieldTypeTest, 
TestFastWriter, SmileWriterTest, HdfsUnloadDistributedZkTest, ReturnFieldsTest, 
RequestHandlersTest, SimpleMLTQParserTest, NotRequiredUniqueKeyTest, 
TestUseDocValuesAsStored, TestSearcherReuse, TestDistributedMissingSort, 
ClassificationUpdateProcessorIntegrationTest, TestConfigSetsAPIZkFailure, 
LeaderElectionIntegrationTest, ChaosMonkeySafeLeaderTest, 
TestFieldTypeCollectionResource, TestCustomSort, TestCloudSchemaless, 
TestSolrQueryParser, DistribJoinFromCollectionTest, NoCacheHeaderTest, 
TestTrackingShardHandlerFactory, HdfsDirectoryTest, TestReload, 
TestRandomFlRTGCloud, SolrIndexSplitterTest, TestConfigSetsAPI, 
PeerSyncWithIndexFingerprintCachingTest, LeaderElectionContextKeyTest, 
DistributedQueryComponentCustomSortTest, 
OverriddenZkACLAndCredentialsProvidersTest, DeleteNodeTest, 
TestSystemIdResolver, CdcrBootstrapTest, BasicDistributedZkTest, OverseerTest, 
ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, 
TestRandomDVFaceting, TestFaceting, TestRecovery, TestHashPartitioner, 
TestRealTimeGet, TestJoin, TestCoreContainer, SimpleFacetsTest, SolrCoreTest, 
SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, 
BadIndexSchemaTest, TestFiltering, TestFunctionQuery, TestLazyCores, 
HighlighterTest, CurrencyFieldXmlFileTest, SimplePostToolTest, 
SpellCheckCollatorTest, CoreAdminHandlerTest, SolrRequestParserTest, 
TestFoldingMultitermQuery, DocValuesTest, XsltUpdateRequestHandlerTest, 
DisMaxRequestHandlerTest, TestQueryUtils, TestQueryTypes, 
FileBasedSpellCheckerTest, TestIndexingPerformance, LoggingHandlerTest, 
IndexSchemaTest, MBeansHandlerTest, JsonLoaderTest, PingRequestHandlerTest, 
TestSolrIndexConfig, BadComponentTest, MultiTermTest, TestBinaryField, 
NumericFieldsTest, OutputWriterTest, TestPostingsSolrHighlighter, 
EchoParamsTest, TestIBSimilarityFactory, TestBM25SimilarityFactory, 
TimeZoneUtilsTest, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, 
PluginInfoTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, 
TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, 
SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, 
DistributedMLTComponentTest, TestCursorMarkWithoutUniqueKey, 
TestSimpleTrackingShardHandler, TestTolerantSearch, TestJettySolrRunner, 
AliasIntegrationTest, AssignTest, AsyncCallRequestStatusResponseTest, 
BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, 
CdcrVersionReplicationTest, CleanupOldIndexTest, 
CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, 
ConfigSetsAPITest, ConnectionManagerTest, DeleteLastCustomShardedReplicaTest, 
DeleteStatusTest, ForceLeaderTest, LeaderInitiatedRecoveryOnShardRestartTest, 
MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, 
OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, 
OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, ReplaceNodeTest, 
RestartWhileUpdatingTest, SaslZkACLProviderTest, 
SharedFSAutoReplicaFailoverTest, SolrXmlInZkTest, TestCloudDeleteByQuery, 
TestCloudRecovery, TestHdfsCloudBackupRestore, TestMiniSolrCloudClusterSSL, 
TestRequestForwarding, TestShortCircuitedRequests, 
TestSolrCloudWithSecureImpersonation, TestTolerantUpdateProcessorRandomCloud, 
HdfsRecoveryZkTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsRecoveryZkTest 
-Dtests.seed=D8E0DA520323B775 -Dtests.slow=true -Dtests.locale=uk-UA 
-Dtests.timezone=Antarctica/Palmer -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | HdfsRecoveryZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 
object(s) that were not released!!! [HdfsTransactionLog]
   [junit4]    > 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
   [junit4]    >        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
   [junit4]    >        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
   [junit4]    >        at 
org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
   [junit4]    >        at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
   [junit4]    >        at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
   [junit4]    >        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
   [junit4]    >        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
   [junit4]    >        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([D8E0DA520323B775]:0)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [531/665 (1!)] on J1 in 52.28s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 62850 lines...]

---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to