Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1357/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.lucene.analysis.ja.TestFactories.test

Error Message:
Unable to parse

Stack Trace:
java.lang.NumberFormatException: Unable to parse
        at 
__randomizedtesting.SeedInfo.seed([30AD7BBC3FFE2065:B8F9446691024D9D]:0)
        at org.apache.lucene.util.ArrayUtil.parse(ArrayUtil.java:94)
        at org.apache.lucene.util.ArrayUtil.parseInt(ArrayUtil.java:83)
        at org.apache.lucene.util.ArrayUtil.parseInt(ArrayUtil.java:51)
        at 
org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilter.incrementToken(DelimitedTermFrequencyTokenFilter.java:67)
        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:731)
        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:642)
        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:540)
        at 
org.apache.lucene.analysis.ja.TestFactories.doTestTokenFilter(TestFactories.java:102)
        at 
org.apache.lucene.analysis.ja.TestFactories.test(TestFactories.java:55)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test

Error Message:
Mismatch in counts between replicas

Stack Trace:
java.lang.AssertionError: Mismatch in counts between replicas
        at 
__randomizedtesting.SeedInfo.seed([2F2FD9C62A3FB22:8AA6C246CC5F96DA]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.RecoveryZkTest.assertShardConsistency(RecoveryZkTest.java:143)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:126)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 3455 lines...]
   [junit4] Suite: org.apache.lucene.analysis.ja.TestFactories
   [junit4]   2> TEST FAIL: useCharFilter=true text='w ?w|v)]? xiB'
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFactories 
-Dtests.method=test -Dtests.seed=30AD7BBC3FFE2065 -Dtests.slow=true 
-Dtests.locale=ar-IQ -Dtests.timezone=Asia/Irkutsk -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   10.5s J1 | TestFactories.test <<<
   [junit4]    > Throwable #1: java.lang.NumberFormatException: Unable to parse
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([30AD7BBC3FFE2065:B8F9446691024D9D]:0)
   [junit4]    >        at 
org.apache.lucene.util.ArrayUtil.parse(ArrayUtil.java:94)
   [junit4]    >        at 
org.apache.lucene.util.ArrayUtil.parseInt(ArrayUtil.java:83)
   [junit4]    >        at 
org.apache.lucene.util.ArrayUtil.parseInt(ArrayUtil.java:51)
   [junit4]    >        at 
org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilter.incrementToken(DelimitedTermFrequencyTokenFilter.java:67)
   [junit4]    >        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:731)
   [junit4]    >        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:642)
   [junit4]    >        at 
org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:540)
   [junit4]    >        at 
org.apache.lucene.analysis.ja.TestFactories.doTestTokenFilter(TestFactories.java:102)
   [junit4]    >        at 
org.apache.lucene.analysis.ja.TestFactories.test(TestFactories.java:55)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/analysis/kuromoji/test/J1/temp/lucene.analysis.ja.TestFactories_30AD7BBC3FFE2065-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, 
sim=RandomSimilarity(queryNorm=true): {dummy=DFR I(ne)LZ(0.3)}, locale=ar-IQ, 
timezone=Asia/Irkutsk
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_131 
(64-bit)/cpus=3,threads=1,free=159074392,total=228589568
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestJapaneseKatakanaStemFilter, UserDictionaryTest, 
TestJapanesePartOfSpeechStopFilterFactory, TestFactories]
   [junit4] Completed [9/21 (1!)] on J1 in 10.62s, 1 test, 1 error <<< FAILURES!

[...truncated 8081 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/init-core-data-001
   [junit4]   2> 843947 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 843949 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 843949 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/tempDir-001
   [junit4]   2> 843949 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 843950 INFO  (Thread-2077) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 843950 INFO  (Thread-2077) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 843953 ERROR (Thread-2077) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 844050 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:55011
   [junit4]   2> 844059 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 844059 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 844061 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@5722b954{/solr,null,AVAILABLE}
   [junit4]   2> 844061 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@72e84e3b{/solr,null,AVAILABLE}
   [junit4]   2> 844062 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@56f36161{HTTP/1.1,[http/1.1]}{127.0.0.1:50762}
   [junit4]   2> 844062 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.e.j.s.Server Started @846221ms
   [junit4]   2> 844062 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=50762}
   [junit4]   2> 844068 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@7ccca708{HTTP/1.1,[http/1.1]}{127.0.0.1:48097}
   [junit4]   2> 844068 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.e.j.s.Server Started @846227ms
   [junit4]   2> 844068 ERROR (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 844068 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=48097}
   [junit4]   2> 844068 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 844068 ERROR (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 844068 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-11T02:50:00.289Z
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 844069 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-06-11T02:50:00.289Z
   [junit4]   2> 844075 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 844076 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 844083 INFO  (jetty-launcher-1369-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55011/solr
   [junit4]   2> 844083 INFO  (jetty-launcher-1369-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55011/solr
   [junit4]   2> 844148 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 844159 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:48097_solr
   [junit4]   2> 844161 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 844164 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.Overseer Overseer 
(id=98117183089672198-127.0.0.1:48097_solr-n_0000000000) starting
   [junit4]   2> 844165 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:50762_solr
   [junit4]   2> 844173 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 844228 INFO  
(zkCallback-1381-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 844281 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50762.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844284 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50762.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844284 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50762.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844285 INFO  (jetty-launcher-1369-thread-2) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/tempDir-001/node2/.
   [junit4]   2> 844318 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:48097_solr
   [junit4]   2> 844336 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 844342 INFO  
(zkCallback-1381-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 844581 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_48097.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844583 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_48097.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844583 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_48097.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 844584 INFO  (jetty-launcher-1369-thread-1) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/tempDir-001/node1/.
   [junit4]   2> 844612 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15c950ee46c0007, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 844619 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 844624 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55011/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 844697 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 844715 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 844717 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 844736 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_serv1.vm3_41786_hdfs____.hn336v/webapp
   [junit4]   2> 845345 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@serv1-vm3:41786
   [junit4]   2> 845760 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 845763 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 845782 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_34925_datanode____.1o5q3i/webapp
   [junit4]   2> 846379 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34925
   [junit4]   2> 846595 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 846596 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 846632 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Extract 
jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_53502_datanode____.gwxji8/webapp
   [junit4]   2> 846731 INFO  (IPC Server handler 0 on 34022) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-6b115b1c-1677-4e5b-bfae-552db076e2d4 node 
DatanodeRegistration(127.0.0.1:57194, 
datanodeUuid=6c3ec7b7-f5ca-4c33-8473-adb1bbe62ea2, infoPort=33319, 
infoSecurePort=0, ipcPort=49964, 
storageInfo=lv=-56;cid=testClusterID;nsid=310015594;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 846731 INFO  (IPC Server handler 0 on 34022) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-2cad2651-3078-499f-8c62-069e84018510 node 
DatanodeRegistration(127.0.0.1:57194, 
datanodeUuid=6c3ec7b7-f5ca-4c33-8473-adb1bbe62ea2, infoPort=33319, 
infoSecurePort=0, ipcPort=49964, 
storageInfo=lv=-56;cid=testClusterID;nsid=310015594;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 847257 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:53502
   [junit4]   2> 847514 INFO  (IPC Server handler 7 on 34022) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-2ee2eccf-eb3a-4117-8c17-3eaaf6bf73b2 node 
DatanodeRegistration(127.0.0.1:49533, 
datanodeUuid=4ba780b8-3a06-4cfa-89e1-e0d2300b5e57, infoPort=35998, 
infoSecurePort=0, ipcPort=36147, 
storageInfo=lv=-56;cid=testClusterID;nsid=310015594;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 847514 INFO  (IPC Server handler 7 on 34022) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-aae61c84-13bd-43bf-b761-6fdf6c782806 node 
DatanodeRegistration(127.0.0.1:49533, 
datanodeUuid=4ba780b8-3a06-4cfa-89e1-e0d2300b5e57, infoPort=35998, 
infoSecurePort=0, ipcPort=36147, 
storageInfo=lv=-56;cid=testClusterID;nsid=310015594;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 847775 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 847780 INFO  (qtp1114205546-8814) [n:127.0.0.1:48097_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 847784 INFO  
(OverseerThreadFactory-3650-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 847907 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 847908 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 847908 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 847908 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 848028 INFO  
(zkCallback-1381-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 848028 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 848937 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 848943 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 848945 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 848946 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using 
configuration from collection recoverytest, trusted=true
   [junit4]   2> 848947 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_48097.solr.core.recoverytest.shard1.replica_n2' (registry 
'solr.core.recoverytest.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 848974 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://serv1-vm3:34022/data
   [junit4]   2> 848974 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 848974 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 848974 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 848974 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica_n2] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/tempDir-001/node1/recoverytest_shard1_replica_n2],
 dataDir=[hdfs://serv1-vm3:34022/data/recoverytest/core_node1/data/]
   [junit4]   2> 848976 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 848986 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 848986 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 848986 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 848986 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 848989 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 848989 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using 
configuration from collection recoverytest, trusted=true
   [junit4]   2> 848990 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_50762.solr.core.recoverytest.shard1.replica_n1' (registry 
'solr.core.recoverytest.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@58f56240
   [junit4]   2> 848990 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://serv1-vm3:34022/data
   [junit4]   2> 848990 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 848990 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 848990 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica_n1] ] Opening new SolrCore at 
[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001/tempDir-001/node2/recoverytest_shard1_replica_n1],
 dataDir=[hdfs://serv1-vm3:34022/data/recoverytest/core_node2/data/]
   [junit4]   2> 848991 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 849000 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 849000 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 849000 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 849118 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 849121 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node1/data
   [junit4]   2> 849130 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 849132 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node2/data
   [junit4]   2> 849166 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node2/data/index
   [junit4]   2> 849182 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 849182 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 849182 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 849250 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://serv1-vm3:34022/data/recoverytest/core_node1/data/index
   [junit4]   2> 849259 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 849259 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 849259 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 849341 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 849428 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 849571 INFO  (IPC Server handler 2 on 34022) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49533 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-6b115b1c-1677-4e5b-bfae-552db076e2d4:NORMAL:127.0.0.1:57194|RBW],
 
ReplicaUC[[DISK]DS-aae61c84-13bd-43bf-b761-6fdf6c782806:NORMAL:127.0.0.1:49533|RBW]]}
 size 0
   [junit4]   2> 849573 INFO  (IPC Server handler 3 on 34022) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57194 is 
added to blk_1073741825_1001 size 72
   [junit4]   2> 849628 INFO  (IPC Server handler 9 on 34022) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57194 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-2ee2eccf-eb3a-4117-8c17-3eaaf6bf73b2:NORMAL:127.0.0.1:49533|RBW],
 
ReplicaUC[[DISK]DS-2cad2651-3078-499f-8c62-069e84018510:NORMAL:127.0.0.1:57194|RBW]]}
 size 0
   [junit4]   2> 849628 INFO  (IPC Server handler 7 on 34022) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49533 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-2ee2eccf-eb3a-4117-8c17-3eaaf6bf73b2:NORMAL:127.0.0.1:49533|RBW],
 
ReplicaUC[[DISK]DS-2cad2651-3078-499f-8c62-069e84018510:NORMAL:127.0.0.1:57194|RBW]]}
 size 0
   [junit4]   2> 849755 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 849756 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 849756 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 849784 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 849784 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 849854 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@2447fc40[recoverytest_shard1_replica_n2] main]
   [junit4]   2> 849868 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 849868 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 849868 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 849873 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 849881 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 849883 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 849884 INFO  
(searcherExecutor-3653-thread-1-processing-n:127.0.0.1:48097_solr 
x:recoverytest_shard1_replica_n2 s:shard1 c:recoverytest) 
[n:127.0.0.1:48097_solr c:recoverytest s:shard1  
x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica_n2] Registered new searcher 
Searcher@2447fc40[recoverytest_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 849885 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1569874935655956480
   [junit4]   2> 849892 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 849892 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 849898 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 849958 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@62eaae09[recoverytest_shard1_replica_n1] main]
   [junit4]   2> 849963 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 849964 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 849965 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 849968 INFO  
(searcherExecutor-3654-thread-1-processing-n:127.0.0.1:50762_solr 
x:recoverytest_shard1_replica_n1 s:shard1 c:recoverytest) 
[n:127.0.0.1:50762_solr c:recoverytest s:shard1  
x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica_n1] Registered new searcher 
Searcher@62eaae09[recoverytest_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 849968 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1569874935742988288
   [junit4]   2> 850003 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850004 INFO  
(zkCallback-1381-thread-2-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850400 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 850400 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 850400 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/
   [junit4]   2> 850401 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica_n2 url=http://127.0.0.1:48097/solr 
START replicas=[http://127.0.0.1:50762/solr/recoverytest_shard1_replica_n1/] 
nUpdates=100
   [junit4]   2> 850402 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica_n2 url=http://127.0.0.1:48097/solr 
DONE.  We have no versions.  sync failed.
   [junit4]   2> 850407 INFO  (qtp786706065-8824) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.c.S.Request [recoverytest_shard1_replica_n1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=1
   [junit4]   2> 850408 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy 
Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 850409 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 850409 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 850423 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/ shard1
   [junit4]   2> 850538 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850539 INFO  
(zkCallback-1381-thread-2-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850585 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.c.ZkController 
I am the leader, no recovery necessary
   [junit4]   2> 850588 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n2] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=2681
   [junit4]   2> 850708 INFO  
(zkCallback-1381-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850710 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 850989 INFO  (qtp786706065-8823) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica_n1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=3082
   [junit4]   2> 851021 INFO  (qtp1114205546-8814) [n:127.0.0.1:48097_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 851121 INFO  
(zkCallback-1381-thread-1-processing-n:127.0.0.1:48097_solr) 
[n:127.0.0.1:48097_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 851123 INFO  
(zkCallback-1380-thread-1-processing-n:127.0.0.1:50762_solr) 
[n:127.0.0.1:50762_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 851789 INFO  
(OverseerCollectionConfigSetProcessor-98117183089672198-127.0.0.1:48097_solr-n_0000000000)
 [n:127.0.0.1:48097_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 852028 INFO  (qtp1114205546-8814) [n:127.0.0.1:48097_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=4248
   [junit4]   2> 852031 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 1200 documents
   [junit4]   2> 852108 DEBUG (qtp1114205546-8817) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://serv1-vm3:34022/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 852124 DEBUG (qtp786706065-8822) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://serv1-vm3:34022/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 852216 INFO  (qtp786706065-8826) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-0
 (1569874937955483648)]} 0 99
   [junit4]   2> 852216 INFO  (qtp1114205546-8815) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1569874937955483648)]} 0 138
   [junit4]   2> 852219 INFO  (qtp786706065-8822) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-0
 (1569874937925074944)]} 0 106
   [junit4]   2> 852220 INFO  (qtp1114205546-8817) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1569874937925074944)]} 0 172
   [junit4]   2> 852233 INFO  (qtp786706065-8827) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-1
 (1569874938102284288)]} 0 13
   [junit4]   2> 852233 INFO  (qtp786706065-8825) [n:127.0.0.1:50762_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48097/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-1
 (1569874938108575744)]} 0 7
   [junit4]   2> 852234 INFO  (qtp1114205546-8818) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1569874938102284288)]} 0 15
   [junit4]   2> 852234 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@56f36161{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 852235 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=673639126
   [junit4]   2> 852235 INFO  (qtp1114205546-8819) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1569874938108575744)]} 0 11
   [junit4]   2> 852236 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, 
tag=null
   [junit4]   2> 852237 ERROR 
(updateExecutor-1375-thread-1-processing-http:////127.0.0.1:50762//solr//recoverytest_shard1_replica_n1
 x:recoverytest_shard1_replica_n2 r:core_node1 n:127.0.0.1:48097_solr s:shard1 
c:recoverytest) [n:127.0.0.1:48097_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica_n2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852238 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, 
tag=null
   [junit4]   2> 852244 WARN  (qtp1114205546-8811) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:50762/solr
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852245 ERROR (qtp1114205546-8811) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:50762/solr/recoverytest_shard1_replica_n1/
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852245 INFO  (qtp1114205546-8811) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica_n1 
coreNodeName=core_node2 on 127.0.0.1:50762_solr into leader-initiated recovery.
   [junit4]   2> 852245 INFO  (qtp1114205546-8811) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1569874938121158656)]} 0 9
   [junit4]   2> 852246 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, 
tag=null
   [junit4]   2> 852246 ERROR 
(updateExecutor-1375-thread-1-processing-http:////127.0.0.1:50762//solr//recoverytest_shard1_replica_n1
 x:recoverytest_shard1_replica_n2 r:core_node1 n:127.0.0.1:48097_solr s:shard1 
c:recoverytest) [n:127.0.0.1:48097_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica_n2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852273 WARN  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:50762/solr
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852273 ERROR (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:50762/solr/recoverytest_shard1_replica_n1/
   [junit4]   2> java.net.SocketException: Broken pipe (Write failed)
   [junit4]   2>        at java.net.SocketOutputStream.socketWrite0(Native 
Method)
   [junit4]   2>        at 
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
   [junit4]   2>        at 
java.net.SocketOutputStream.write(SocketOutputStream.java:155)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:126)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:138)
   [junit4]   2>        at 
org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:146)
   [junit4]   2>        at 
org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:205)
   [junit4]   2>        at 
org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:159)
   [junit4]   2>        at 
org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:162)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:237)
   [junit4]   2>        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:122)
   [junit4]   2>        at 
org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852273 INFO  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1569874938129547264)]} 0 34
   [junit4]   2> 852274 ERROR 
(updateExecutor-1375-thread-1-processing-http:////127.0.0.1:50762//solr//recoverytest_shard1_replica_n1
 x:recoverytest_shard1_replica_n2 r:core_node1 n:127.0.0.1:48097_solr s:shard1 
c:recoverytest) [n:127.0.0.1:48097_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica_n2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> org.apache.http.conn.HttpHostConnectException: Connect to 
127.0.0.1:50762 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]   2>        at 
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused 
(Connection refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]   2>        ... 16 more
   [junit4]   2> 852275 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[2F2FD9C62A3FB22]) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, 
tag=null
   [junit4]   2> 852276 WARN  (qtp1114205546-8814) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:50762/solr
   [junit4]   2> org.apache.http.conn.HttpHostConnectException: Connect to 
127.0.0.1:50762 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]   2>        at 
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused 
(Connection refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]   2>        ... 16 more
   [junit4]   2> 852276 ERROR (qtp1114205546-8814) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:50762/solr/recoverytest_shard1_replica_n1/
   [junit4]   2> java.net.ConnectException: Connection refused (Connection 
refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]   2>        at 
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 852277 INFO  (qtp1114205546-8814) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1569874938159955968)]} 0 3
   [junit4]   2> 852278 INFO  (coreCloseExecutor-3663-thread-1) 
[n:127.0.0.1:50762_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica_n1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@29b86d62
   [junit4]   2> 852279 ERROR 
(updateExecutor-1375-thread-1-processing-http:////127.0.0.1:50762//solr//recoverytest_shard1_replica_n1
 x:recoverytest_shard1_replica_n2 r:core_node1 n:127.0.0.1:48097_solr s:shard1 
c:recoverytest) [n:127.0.0.1:48097_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica_n2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> org.apache.http.conn.HttpHostConnectException: Connect to 
127.0.0.1:50762 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]   2>        at 
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]   2>        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]   2>        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]   2>        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:326)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:161)
   [junit4]   2>        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused 
(Connection refused)
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]   2>        ... 16 more
   [junit4]   2> 852280 WARN  (qtp1114205546-8816) [n:127.0.0.1:48097_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:50762/solr
   [junit4]   2> org.apache.http.conn.HttpHostConnectException: Connect to 
127.0.0.1:50762 [/127.0.0.1] failed: Connection refused (Connection refused)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]   2>        at 
org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]   2>        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]   2>        at org.apache.http

[...truncated too long message...]

ethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2>        at 
com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.teardownClass(HdfsRecoveryZkTest.java:53)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 54 more
   [junit4]   2> 870442 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[2F2FD9C62A3FB22]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:55011 55011
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_2F2FD9C62A3FB22-001
   [junit4]   2> Jun 11, 2017 2:50:26 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 27 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{rnd_b=PostingsFormat(name=Asserting), _version_=FSTOrd50, a_t=FSTOrd50, 
a_i=PostingsFormat(name=Asserting), id=PostingsFormat(name=Asserting)}, 
docValues:{}, maxPointsInLeafNode=993, maxMBSortInHeap=7.454645921394048, 
sim=RandomSimilarity(queryNorm=false): {}, locale=ar-IQ, timezone=Africa/Asmara
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_131 
(64-bit)/cpus=3,threads=2,free=277247496,total=446693376
   [junit4]   2> NOTE: All tests run in this JVM: [ExplicitHLLTest, 
TestSubQueryTransformerCrossCore, BlockCacheTest, DistributedQueueTest, 
BigEndianAscendingWordSerializerTest, RuleEngineTest, TestZkChroot, 
TestConfigReload, TestQueryTypes, TestCloudInspectUtil, 
TestJsonFacetRefinement, TestClassNameShortening, ZkStateWriterTest, 
DistanceFunctionTest, TestExclusionRuleCollectionAccess, 
TestMaxTokenLenTokenizer, TestFuzzyAnalyzedSuggestions, 
TestSimpleQParserPlugin, TestFastWriter, TestManagedStopFilterFactory, 
HdfsChaosMonkeySafeLeaderTest, TestTestInjection, 
UpdateRequestProcessorFactoryTest, RemoteQueryErrorTest, 
TestGraphTermsQParserPlugin, TestSolr4Spatial, ActionThrottleTest, 
TestFieldCache, UUIDFieldTest, TestWriterPerf, CacheHeaderTest, JsonLoaderTest, 
HLLSerializationTest, ParsingFieldUpdateProcessorsTest, CircularListTest, 
TestShardHandlerFactory, TestAnalyzedSuggestions, IndexSchemaTest, 
TestSolrConfigHandlerConcurrent, ChaosMonkeyNothingIsSafeWithPullReplicasTest, 
TestManagedSchemaThreadSafety, HdfsLockFactoryTest, 
ClassificationUpdateProcessorFactoryTest, InfixSuggestersTest, 
SolrShardReporterTest, TestCollectionAPI, BadComponentTest, 
SpellCheckCollatorWithCollapseTest, TestEmbeddedSolrServerConstructors, 
CustomCollectionTest, TestCoreDiscovery, JSONWriterTest, 
TestSlowCompositeReaderWrapper, CoreSorterTest, 
TestOverriddenPrefixQueryForCustomFieldType, RollingRestartTest, 
TestConfigSetProperties, CloudExitableDirectoryReaderTest, 
ChaosMonkeyNothingIsSafeTest, DistributedTermsComponentTest, TestCloudRecovery, 
TestSizeLimitedDistributedMap, StatsComponentTest, TestValueSourceCache, 
LeaderFailoverAfterPartitionTest, TestDocBasedVersionConstraints, 
TestXmlQParser, TestDefaultStatsCache, BufferStoreTest, 
SignatureUpdateProcessorFactoryTest, TestSolrIndexConfig, 
TestFreeTextSuggestions, ResponseHeaderTest, TestReload, 
TestLegacyNumericUtils, LeaderInitiatedRecoveryOnCommitTest, PrimUtilsTest, 
SolrMetricManagerTest, MergeStrategyTest, 
PeerSyncWithIndexFingerprintCachingTest, TestCodecSupport, TestSolr4Spatial2, 
DocExpirationUpdateProcessorFactoryTest, SliceStateTest, CSVRequestHandlerTest, 
HdfsNNFailoverTest, TestDistribDocBasedVersion, BlockJoinFacetSimpleTest, 
BasicZkTest, TestPushWriter, TestComponentsName, TestScoreJoinQPNoScore, 
TestAuthorizationFramework, TestMultiValuedNumericRangeQuery, 
DefaultValueUpdateProcessorTest, TestLMJelinekMercerSimilarityFactory, 
DistributedSuggestComponentTest, SystemInfoHandlerTest, 
TestMaxScoreQueryParser, TestRequestStatusCollectionAPI, 
TestUseDocValuesAsStored, PreAnalyzedFieldManagedSchemaCloudTest, 
DistributedIntervalFacetingTest, OverriddenZkACLAndCredentialsProvidersTest, 
TestReversedWildcardFilterFactory, TestBulkSchemaConcurrent, TestRealTimeGet, 
MigrateRouteKeyTest, AutoCommitTest, MoveReplicaTest, TestCrossCoreJoin, 
TestRestManager, RequestLoggingTest, SortSpecParsingTest, 
HdfsTlogReplayBufferedWhileIndexingTest, EchoParamsTest, 
TestDelegationWithHadoopAuth, HdfsRecoveryZkTest]
   [junit4] Completed [219/724 (1!)] on J1 in 36.06s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 45080 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to