Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1253/

3 tests failed.
FAILED:  
org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
        at 
__randomizedtesting.SeedInfo.seed([EE232EE8E5BD054B:57505A969995D5BE]:0)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
        at 
org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction(ExtractingRequestHandlerTest.java:128)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: 
xpath=//doc[1]/str[.='simple3']
        xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int 
name="QTime">0</int></lst><result name="response" numFound="2" 
start="0"><doc><arr 
name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome
 to 
Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html;
 charset=ISO-8859-1</str></arr><arr 
name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str 
name="id">simple2</str><arr name="stream_size"><str>365</str></arr><arr 
name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr
 name="stream_content_type"><str>application/xml</str></arr><arr 
name="stream_name"><str>simple.html</str></arr><arr 
name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr
 name="t_dc_title"><str>Welcome to Solr</str></arr><arr 
name="t_content_encoding"><str>ISO-8859-1</str></arr><arr 
name="title"><str>Welcome to Solr</str></arr><arr 
name="t_abcxyz"><str>text/html; charset=ISO-8859-1</str></arr><arr 
name="t_content"><str> 
 
  
  
  
  
  
  
  
  
  
 Welcome to Solr 
 
 
 
  Here is some text
 

 distinct
words 

 Here is some text in a div 

 This has a  link . 

  </str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int 
name="intDefault">42</int><date 
name="timestamp">2017-03-03T20:24:28.372Z</date></doc><doc><arr 
name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome
 to 
Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html;
 charset=ISO-8859-1</str></arr><arr 
name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str 
name="id">simple3</str><arr name="stream_size"><str>365</str></arr><arr 
name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr
 name="stream_content_type"><str>application/xml</str></arr><arr 
name="stream_name"><str>simple.html</str></arr><arr 
name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr
 name="t_dc_title"><str>Welcome to Solr</str></arr><arr 
name="t_content_encoding"><str>ISO-8859-1</str></arr><arr 
name="title"><str>Welcome to Solr</str></arr><arr 
name="t_content_type"><str>text/html; charset=ISO-8859-1</str></arr><arr 
name="t_content"><str> 
 
  
  
  
  
  
  
  
  
  
 Welcome to Solr 
 
 
 
  Here is some text
 

 distinct
words 

 Here is some text in a div 

 This has a  link . 

  </str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int 
name="intDefault">42</int><date 
name="timestamp">2017-03-03T20:24:28.487Z</date></doc></result>
</response>

        request was:q=t_href:http&qt=standard&start=0&rows=20&version=2.2
        at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
        ... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
  at sun.reflect.GeneratedConstructorAccessor207.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:937)  at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
        at sun.reflect.GeneratedConstructorAccessor207.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937)
        at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
        at sun.reflect.GeneratedMethodAccessor108.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.update.TestInPlaceUpdatesStandalone

Error Message:
Clean up static fields (in @AfterClass?), your test seems to hang on to 
approximately 22,002,080 bytes (threshold is 10,485,760). Field reference sizes 
(counted individually):   - 22,001,752 bytes, private static 
org.apache.solr.client.solrj.SolrClient 
org.apache.solr.update.TestInPlaceUpdatesStandalone.client   - 192 bytes, 
public static org.junit.rules.TestRule 
org.apache.solr.SolrTestCaseJ4.solrClassRules   - 128 bytes, private static 
java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp   - 64 bytes, 
private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName   - 48 
bytes, private static java.util.Map 
org.apache.solr.SolrTestCaseJ4.savedClassLogLevels

Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?), 
your test seems to hang on to approximately 22,002,080 bytes (threshold is 
10,485,760). Field reference sizes (counted individually):
  - 22,001,752 bytes, private static org.apache.solr.client.solrj.SolrClient 
org.apache.solr.update.TestInPlaceUpdatesStandalone.client
  - 192 bytes, public static org.junit.rules.TestRule 
org.apache.solr.SolrTestCaseJ4.solrClassRules
  - 128 bytes, private static java.lang.String 
org.apache.solr.SolrTestCaseJ4.factoryProp
  - 64 bytes, private static java.lang.String 
org.apache.solr.SolrTestCaseJ4.coreName
  - 48 bytes, private static java.util.Map 
org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
        at __randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
        at 
com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:170)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 13145 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/init-core-data-001
   [junit4]   2> 3619508 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 3619509 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3619510 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001
   [junit4]   2> 3619510 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3619510 INFO  (Thread-40689) [    ] o.a.s.c.ZkTestServer 
client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3619510 INFO  (Thread-40689) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 3619610 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:56173
   [junit4]   2> 3619615 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3619615 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3619616 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@5f74f634{/solr,null,AVAILABLE}
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@50c23115{HTTP/1.1,[http/1.1]}{127.0.0.1:38084}
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.e.j.s.Server Started @3621959ms
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=38084}
   [junit4]   2> 3619617 ERROR (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.0.0
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3619617 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T20:08:56.185Z
   [junit4]   2> 3619618 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@633c7b87{/solr,null,AVAILABLE}
   [junit4]   2> 3619618 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@3390b519{HTTP/1.1,[http/1.1]}{127.0.0.1:56345}
   [junit4]   2> 3619618 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.e.j.s.Server Started @3621960ms
   [junit4]   2> 3619618 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=56345}
   [junit4]   2> 3619618 ERROR (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3619619 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.0.0
   [junit4]   2> 3619619 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3619619 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3619619 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T20:08:56.187Z
   [junit4]   2> 3619619 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3619625 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3619631 INFO  (jetty-launcher-3095-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56173/solr
   [junit4]   2> 3619632 INFO  (jetty-launcher-3095-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56173/solr
   [junit4]   2> 3619648 INFO  (jetty-launcher-3095-thread-2) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3619648 INFO  (jetty-launcher-3095-thread-1) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3619648 INFO  (jetty-launcher-3095-thread-2) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:56345_solr
   [junit4]   2> 3619649 INFO  (jetty-launcher-3095-thread-1) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:38084_solr
   [junit4]   2> 3619649 INFO  (jetty-launcher-3095-thread-2) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.Overseer Overseer 
(id=97555037295673350-127.0.0.1:56345_solr-n_0000000000) starting
   [junit4]   2> 3619650 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3619650 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3619664 INFO  (jetty-launcher-3095-thread-2) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:56345_solr
   [junit4]   2> 3619665 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 3619665 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 3619687 INFO  (jetty-launcher-3095-thread-1) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node1/.
   [junit4]   2> 3619743 INFO  (jetty-launcher-3095-thread-2) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node2/.
   [junit4]   2> 3620577 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3620577 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56173/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 3620607 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 3620611 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3620613 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3620623 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_44215_hdfs____.gzokjk/webapp
   [junit4]   2> 3620944 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44215
   [junit4]   2> 3621005 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3621006 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3621014 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_56802_datanode____wjv3og/webapp
   [junit4]   2> 3621384 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56802
   [junit4]   2> 3621419 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3621420 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3621429 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_45051_datanode____.4pblym/webapp
   [junit4]   2> 3621471 INFO  (IPC Server handler 4 on 35882) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-ad324f86-df3c-4c65-acc5-6dc0ec209410 node 
DatanodeRegistration(127.0.0.1:38037, 
datanodeUuid=31a98948-f154-46e5-9479-a25868a05ce1, infoPort=52146, 
infoSecurePort=0, ipcPort=32908, 
storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3621471 INFO  (IPC Server handler 4 on 35882) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a node 
DatanodeRegistration(127.0.0.1:38037, 
datanodeUuid=31a98948-f154-46e5-9479-a25868a05ce1, infoPort=52146, 
infoSecurePort=0, ipcPort=32908, 
storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3621848 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [    ] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45051
   [junit4]   2> 3621930 INFO  (IPC Server handler 8 on 35882) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-79f997cb-cf84-4288-8327-334d4505c388 node 
DatanodeRegistration(127.0.0.1:60714, 
datanodeUuid=7312cf2a-11eb-4dad-86d2-216f4285f9ce, infoPort=40459, 
infoSecurePort=0, ipcPort=33582, 
storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3621930 INFO  (IPC Server handler 8 on 35882) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-2e2f9472-8949-4754-806b-72c93dde0352 node 
DatanodeRegistration(127.0.0.1:60714, 
datanodeUuid=7312cf2a-11eb-4dad-86d2-216f4285f9ce, infoPort=40459, 
infoSecurePort=0, ipcPort=33582, 
storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3622031 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D131E1FDF367CB3]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 3622032 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 3622033 INFO  
(OverseerThreadFactory-8887-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 3622137 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 3622137 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 3622242 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3622242 INFO  
(zkCallback-3107-thread-3-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3623147 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 3623149 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 3623165 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica2] Schema name=minimal
   [junit4]   2> 3623166 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica1] Schema name=minimal
   [junit4]   2> 3623167 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3623168 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica2' using configuration from 
collection recoverytest
   [junit4]   2> 3623168 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35882/data
   [junit4]   2> 3623168 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3623168 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 3623168 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node1/recoverytest_shard1_replica2],
 dataDir=[hdfs://localhost:35882/data/recoverytest/core_node2/data/]
   [junit4]   2> 3623168 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f70fb2c
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica1' using configuration from 
collection recoverytest
   [junit4]   2> 3623169 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35882/data
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node2/recoverytest_shard1_replica1],
 dataDir=[hdfs://localhost:35882/data/recoverytest/core_node1/data/]
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f70fb2c
   [junit4]   2> 3623169 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 3623174 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3623174 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3623174 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3623174 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3623180 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3623182 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3623182 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node2/data
   [junit4]   2> 3623184 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node1/data
   [junit4]   2> 3623202 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node2/data/index
   [junit4]   2> 3623204 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35882/data/recoverytest/core_node1/data/index
   [junit4]   2> 3623208 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3623208 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3623209 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3623209 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3623211 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3623212 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3623249 INFO  (IPC Server handler 8 on 35882) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60714 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW],
 
ReplicaUC[[DISK]DS-79f997cb-cf84-4288-8327-334d4505c388:NORMAL:127.0.0.1:60714|RBW]]}
 size 0
   [junit4]   2> 3623250 INFO  (IPC Server handler 9 on 35882) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38037 is 
added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-79f997cb-cf84-4288-8327-334d4505c388:NORMAL:127.0.0.1:60714|RBW],
 
ReplicaUC[[DISK]DS-ad324f86-df3c-4c65-acc5-6dc0ec209410:NORMAL:127.0.0.1:38037|FINALIZED]]}
 size 0
   [junit4]   2> 3623251 INFO  (IPC Server handler 0 on 35882) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38037 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-2e2f9472-8949-4754-806b-72c93dde0352:NORMAL:127.0.0.1:60714|RBW],
 
ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW]]}
 size 0
   [junit4]   2> 3623253 INFO  (IPC Server handler 6 on 35882) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60714 is 
added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-2e2f9472-8949-4754-806b-72c93dde0352:NORMAL:127.0.0.1:60714|RBW],
 
ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW]]}
 size 0
   [junit4]   2> 3623307 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 3623308 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3623308 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 3623321 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 3623321 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 3623334 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@7082e0ca[recoverytest_shard1_replica2] main]
   [junit4]   2> 3623335 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 3623335 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3623335 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 3623335 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 3623335 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3623335 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 3623336 INFO  
(searcherExecutor-8892-thread-1-processing-n:127.0.0.1:38084_solr 
x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica2] Registered new searcher 
Searcher@7082e0ca[recoverytest_shard1_replica2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3623338 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1560880600743673856
   [junit4]   2> 3623343 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 3623345 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 3623345 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 3623350 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@71ffecbc[recoverytest_shard1_replica1] main]
   [junit4]   2> 3623350 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 3623351 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3623351 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 3623352 INFO  
(searcherExecutor-8893-thread-1-processing-n:127.0.0.1:56345_solr 
x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica1] Registered new searcher 
Searcher@71ffecbc[recoverytest_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3623353 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1560880600759402496
   [junit4]   2> 3623444 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3623444 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3623843 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3623844 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3623844 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/
   [junit4]   2> 3623844 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:38084/solr 
START replicas=[http://127.0.0.1:56345/solr/recoverytest_shard1_replica1/] 
nUpdates=100
   [junit4]   2> 3623844 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:38084/solr 
DONE.  We have no versions.  sync failed.
   [junit4]   2> 3623845 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.c.S.Request [recoverytest_shard1_replica1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 3623845 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 3623845 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 3623845 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 3623849 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/ shard1
   [junit4]   2> 3623949 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3623949 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3623999 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 3624001 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=1863
   [junit4]   2> 3624102 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3624102 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3624360 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=2223
   [junit4]   2> 3624362 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 3624459 INFO  
(zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) 
[n:127.0.0.1:56345_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3624459 INFO  
(zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) 
[n:127.0.0.1:38084_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3625363 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=3330
   [junit4]   2> 3625364 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[D131E1FDF367CB3]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 3000 documents
   [junit4]   2> 3625372 DEBUG (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://localhost:35882/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 3625375 DEBUG (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://localhost:35882/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 3625379 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0
 (1560880602873331712)]} 0 5
   [junit4]   2> 3625379 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1560880602873331712)]} 0 10
   [junit4]   2> 3625380 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1
 (1560880602883817472)]} 0 0
   [junit4]   2> 3625380 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1560880602883817472)]} 0 0
   [junit4]   2> 3625381 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2
 (1560880602885914624)]} 0 0
   [junit4]   2> 3625381 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1560880602885914624)]} 0 0
   [junit4]   2> 3625383 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3
 (1560880602886963200)]} 0 0
   [junit4]   2> 3625383 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1560880602886963200)]} 0 0
   [junit4]   2> 3625384 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4
 (1560880602888011776)]} 0 0
   [junit4]   2> 3625384 INFO  (qtp141765294-57534) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 
(1560880602888011776)]} 0 1
   [junit4]   2> 3625384 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0
 (1560880602875428864)]} 0 0
   [junit4]   2> 3625385 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1560880602875428864)]} 0 14
   [junit4]   2> 3625385 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5
 (1560880602890108928)]} 0 0
   [junit4]   2> 3625386 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 
(1560880602890108928)]} 0 1
   [junit4]   2> 3625386 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1
 (1560880602890108929)]} 0 0
   [junit4]   2> 3625387 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1560880602890108929)]} 0 1
   [junit4]   2> 3625387 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6
 (1560880602891157504)]} 0 0
   [junit4]   2> 3625387 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 
(1560880602891157504)]} 0 1
   [junit4]   2> 3625388 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7
 (1560880602893254656)]} 0 0
   [junit4]   2> 3625389 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 
(1560880602893254656)]} 0 0
   [junit4]   2> 3625390 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8
 (1560880602894303232)]} 0 0
   [junit4]   2> 3625390 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 
(1560880602894303232)]} 0 0
   [junit4]   2> 3625391 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9
 (1560880602895351808)]} 0 0
   [junit4]   2> 3625391 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 
(1560880602895351808)]} 0 0
   [junit4]   2> 3625392 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-4
 (-1560880602896400384)]} 0 0
   [junit4]   2> 3625392 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 
(-1560880602896400384)]} 0 1
   [junit4]   2> 3625393 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10
 (1560880602898497536)]} 0 0
   [junit4]   2> 3625394 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 
(1560880602898497536)]} 0 0
   [junit4]   2> 3625395 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11
 (1560880602899546112)]} 0 0
   [junit4]   2> 3625395 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 
(1560880602899546112)]} 0 0
   [junit4]   2> 3625396 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12
 (1560880602900594688)]} 0 0
   [junit4]   2> 3625396 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 
(1560880602900594688)]} 0 0
   [junit4]   2> 3625397 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13
 (1560880602901643264)]} 0 0
   [junit4]   2> 3625397 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 
(1560880602901643264)]} 0 0
   [junit4]   2> 3625398 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14
 (1560880602902691840)]} 0 0
   [junit4]   2> 3625398 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2
 (1560880602892206080)]} 0 0
   [junit4]   2> 3625398 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 
(1560880602902691840)]} 0 0
   [junit4]   2> 3625398 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1560880602892206080)]} 0 10
   [junit4]   2> 3625400 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-5
 (-1560880602904788992)]} 0 0
   [junit4]   2> 3625400 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3
 (1560880602904788993)]} 0 0
   [junit4]   2> 3625400 INFO  (qtp141765294-57534) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 
(-1560880602904788992)]} 0 1
   [junit4]   2> 3625400 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 
(1560880602904788993)]} 0 1
   [junit4]   2> 3625403 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15
 (1560880602906886144)]} 0 0
   [junit4]   2> 3625403 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 
(1560880602906886144)]} 0 2
   [junit4]   2> 3625403 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4
 (1560880602906886145)]} 0 0
   [junit4]   2> 3625404 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 
(1560880602906886145)]} 0 3
   [junit4]   2> 3625404 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6
 (-1560880602910031872)]} 0 0
   [junit4]   2> 3625405 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 
(-1560880602910031872)]} 0 1
   [junit4]   2> 3625405 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5
 (1560880602910031873)]} 0 0
   [junit4]   2> 3625405 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 
(1560880602910031873)]} 0 0
   [junit4]   2> 3625406 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16
 (1560880602911080448)]} 0 0
   [junit4]   2> 3625406 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 
(1560880602911080448)]} 0 1
   [junit4]   2> 3625406 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6
 (1560880602911080449)]} 0 0
   [junit4]   2> 3625407 INFO  (qtp141765294-57534) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 
(1560880602911080449)]} 0 1
   [junit4]   2> 3625407 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17
 (1560880602913177600)]} 0 0
   [junit4]   2> 3625407 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 
(1560880602913177600)]} 0 0
   [junit4]   2> 3625408 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7
 (1560880602913177601)]} 0 0
   [junit4]   2> 3625408 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 
(1560880602913177601)]} 0 1
   [junit4]   2> 3625409 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18
 (1560880602914226176)]} 0 0
   [junit4]   2> 3625409 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 
(1560880602914226176)]} 0 1
   [junit4]   2> 3625409 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8
 (1560880602915274752)]} 0 0
   [junit4]   2> 3625410 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 
(1560880602915274752)]} 0 1
   [junit4]   2> 3625410 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8
 (-1560880602915274753)]} 0 0
   [junit4]   2> 3625410 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 
(-1560880602915274753)]} 0 1
   [junit4]   2> 3625411 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9
 (1560880602916323328)]} 0 0
   [junit4]   2> 3625411 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 
(1560880602916323328)]} 0 0
   [junit4]   2> 3625412 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19
 (1560880602917371904)]} 0 0
   [junit4]   2> 3625412 INFO  (qtp141765294-57534) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 
(1560880602917371904)]} 0 1
   [junit4]   2> 3625412 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-4
 (-1560880602917371905)]} 0 0
   [junit4]   2> 3625412 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 
(-1560880602917371905)]} 0 1
   [junit4]   2> 3625413 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-20
 (1560880602918420480)]} 0 0
   [junit4]   2> 3625413 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-20 
(1560880602918420480)]} 0 0
   [junit4]   2> 3625413 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10
 (1560880602919469056)]} 0 0
   [junit4]   2> 3625414 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 
(1560880602919469056)]} 0 0
   [junit4]   2> 3625414 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-21
 (1560880602919469057)]} 0 0
   [junit4]   2> 3625414 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-21 
(1560880602919469057)]} 0 1
   [junit4]   2> 3625415 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9
 (-1560880602921566208)]} 0 0
   [junit4]   2> 3625415 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 
(-1560880602921566208)]} 0 0
   [junit4]   2> 3625416 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-22
 (1560880602922614784)]} 0 0
   [junit4]   2> 3625417 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-22 
(1560880602922614784)]} 0 0
   [junit4]   2> 3625417 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-23
 (1560880602923663360)]} 0 0
   [junit4]   2> 3625418 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-23 
(1560880602923663360)]} 0 0
   [junit4]   2> 3625419 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-24
 (1560880602924711936)]} 0 0
   [junit4]   2> 3625419 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-24 
(1560880602924711936)]} 0 0
   [junit4]   2> 3625420 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-25
 (1560880602925760512)]} 0 0
   [junit4]   2> 3625420 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-25 
(1560880602925760512)]} 0 0
   [junit4]   2> 3625421 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-26
 (1560880602926809088)]} 0 0
   [junit4]   2> 3625421 INFO  (qtp141765294-57544) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-26 
(1560880602926809088)]} 0 0
   [junit4]   2> 3625422 INFO  (qtp1310391835-57539) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-27
 (1560880602927857664)]} 0 0
   [junit4]   2> 3625422 INFO  (qtp141765294-57546) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-27 
(1560880602927857664)]} 0 0
   [junit4]   2> 3625423 INFO  (qtp1310391835-57549) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-12
 (-1560880602928906240)]} 0 0
   [junit4]   2> 3625427 INFO  (qtp141765294-57535) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 
(-1560880602928906240)]} 0 5
   [junit4]   2> 3625428 INFO  (qtp1310391835-57543) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11
 (1560880602920517632)]} 0 0
   [junit4]   2> 3625428 INFO  (qtp141765294-57540) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 
(1560880602920517632)]} 0 13
   [junit4]   2> 3625428 INFO  (qtp1310391835-57547) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-28
 (1560880602935197696)]} 0 0
   [junit4]   2> 3625429 INFO  (qtp141765294-57538) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-28 
(1560880602935197696)]} 0 1
   [junit4]   2> 3625429 INFO  (qtp1310391835-57550) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12
 (1560880602936246272)]} 0 0
   [junit4]   2> 3625430 INFO  (qtp141765294-57534) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 
(1560880602936246272)]} 0 0
   [junit4]   2> 3625430 INFO  (qtp1310391835-57536) [n:127.0.0.1:56345_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-29
 (1560880602936246273)]} 0 0
   [junit4]   2> 3625430 INFO  (qtp141765294-57542) [n:127.0.0.1:38084_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2

[...truncated too long message...]

tionFailedError: Clean up static fields (in @AfterClass?), your test seems to 
hang on to approximately 22,002,080 bytes (threshold is 10,485,760). Field 
reference sizes (counted individually):
   [junit4]    >   - 22,001,752 bytes, private static 
org.apache.solr.client.solrj.SolrClient 
org.apache.solr.update.TestInPlaceUpdatesStandalone.client
   [junit4]    >   - 192 bytes, public static org.junit.rules.TestRule 
org.apache.solr.SolrTestCaseJ4.solrClassRules
   [junit4]    >   - 128 bytes, private static java.lang.String 
org.apache.solr.SolrTestCaseJ4.factoryProp
   [junit4]    >   - 64 bytes, private static java.lang.String 
org.apache.solr.SolrTestCaseJ4.coreName
   [junit4]    >   - 48 bytes, private static java.util.Map 
org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [693/693 (2!)] on J2 in 171.37s, 18 tests, 1 failure <<< 
FAILURES!

[...truncated 5234 lines...]
   [junit4] Suite: 
org.apache.solr.handler.extraction.ExtractingRequestHandlerTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-cell/test/J1/temp/solr.handler.extraction.ExtractingRequestHandlerTest_EE232EE8E5BD054B-001/init-core-data-001
   [junit4]   2> log4j:WARN No appenders could be found for logger 
(org.apache.solr.SolrTestCaseJ4).
   [junit4]   2> log4j:WARN Please initialize the log4j system properly.
   [junit4]   2> log4j:WARN See 
http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=ExtractingRequestHandlerTest -Dtests.method=testExtraction 
-Dtests.seed=EE232EE8E5BD054B -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt
 -Dtests.locale=fr-BE -Dtests.timezone=SystemV/PST8 -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   2.33s J1 | ExtractingRequestHandlerTest.testExtraction <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during 
query
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([EE232EE8E5BD054B:57505A969995D5BE]:0)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
   [junit4]    >        at 
org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction(ExtractingRequestHandlerTest.java:128)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: 
xpath=//doc[1]/str[.='simple3']
   [junit4]    >        xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int 
name="QTime">0</int></lst><result name="response" numFound="2" 
start="0"><doc><arr 
name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome
 to 
Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html;
 charset=ISO-8859-1</str></arr><arr 
name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str 
name="id">simple2</str><arr name="stream_size"><str>365</str></arr><arr 
name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr
 name="stream_content_type"><str>application/xml</str></arr><arr 
name="stream_name"><str>simple.html</str></arr><arr 
name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr
 name="t_dc_title"><str>Welcome to Solr</str></arr><arr 
name="t_content_encoding"><str>ISO-8859-1</str></arr><arr 
name="title"><str>Welcome to Solr</str></arr><arr 
name="t_abcxyz"><str>text/html; charset=ISO-8859-1</str></arr><arr 
name="t_content"><str> 
   [junit4]    >  
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >  Welcome to Solr 
   [junit4]    >  
   [junit4]    >  
   [junit4]    >  
   [junit4]    >   Here is some text
   [junit4]    >  
   [junit4]    >  distinct
   [junit4]    > words 
   [junit4]    >  Here is some text in a div 
   [junit4]    >  This has a  link . 
   [junit4]    >   </str></arr><arr 
name="multiDefault"><str>muLti-Default</str></arr><int 
name="intDefault">42</int><date 
name="timestamp">2017-03-03T20:24:28.372Z</date></doc><doc><arr 
name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome
 to 
Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html;
 charset=ISO-8859-1</str></arr><arr 
name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str 
name="id">simple3</str><arr name="stream_size"><str>365</str></arr><arr 
name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr
 name="stream_content_type"><str>application/xml</str></arr><arr 
name="stream_name"><str>simple.html</str></arr><arr 
name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr
 name="t_dc_title"><str>Welcome to Solr</str></arr><arr 
name="t_content_encoding"><str>ISO-8859-1</str></arr><arr 
name="title"><str>Welcome to Solr</str></arr><arr 
name="t_content_type"><str>text/html; charset=ISO-8859-1</str></arr><arr 
name="t_content"><str> 
   [junit4]    >  
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >   
   [junit4]    >  Welcome to Solr 
   [junit4]    >  
   [junit4]    >  
   [junit4]    >  
   [junit4]    >   Here is some text
   [junit4]    >  
   [junit4]    >  distinct
   [junit4]    > words 
   [junit4]    >  Here is some text in a div 
   [junit4]    >  This has a  link . 
   [junit4]    >   </str></arr><arr 
name="multiDefault"><str>muLti-Default</str></arr><int 
name="intDefault">42</int><date 
name="timestamp">2017-03-03T20:24:28.487Z</date></doc></result>
   [junit4]    > </response>
   [junit4]    >        request 
was:q=t_href:http&qt=standard&start=0&rows=20&version=2.2
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
   [junit4]    >        ... 41 more
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-cell/test/J1/temp/solr.handler.extraction.ExtractingRequestHandlerTest_EE232EE8E5BD054B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{extractionLiteralMV=BlockTreeOrds(blocksize=128), 
stream_content_type=BlockTreeOrds(blocksize=128), 
multiDefault=Lucene50(blocksize=128), subject=Lucene50(blocksize=128), 
intDefault=FST50, t_href=FST50, title=Lucene50(blocksize=128), 
extractedContent=BlockTreeOrds(blocksize=128), stream_name=FST50, 
t_X-Parsed-By=BlockTreeOrds(blocksize=128), bar_s=BlockTreeOrds(blocksize=128), 
Content-Encoding=BlockTreeOrds(blocksize=128), 
t_dc_title=BlockTreeOrds(blocksize=128), 
t_content_type=BlockTreeOrds(blocksize=128), extractedProducer=FST50, 
id=BlockTreeOrds(blocksize=128), 
text=PostingsFormat(name=LuceneVarGapFixedInterval), Content-Type=FST50, 
timestamp=FST50, stream_size=Lucene50(blocksize=128), 
extractedAuthor=BlockTreeOrds(blocksize=128), extractedKeywords=FST50, 
t_content_encoding=Lucene50(blocksize=128), 
stream_source_info=PostingsFormat(name=LuceneVarGapFixedInterval), 
t_content=FST50, resourceName=BlockTreeOrds(blocksize=128), 
foo_t=BlockTreeOrds(blocksize=128), foo_s=FST50, 
title_lettertok=BlockTreeOrds(blocksize=128), 
extractedCreator=Lucene50(blocksize=128), t_meta=Lucene50(blocksize=128), 
t_abcxyz=FST50, t_dc:title=Lucene50(blocksize=128), 
wdf_nocase=BlockTreeOrds(blocksize=128), extractedDate=FST50, div_t=FST50, 
title_stemmed=Lucene50(blocksize=128), anchor_t=FST50, 
t_x_parsed_by=PostingsFormat(name=LuceneVarGapFixedInterval), 
zot_s=BlockTreeOrds(blocksize=128), extractionLiteral=Lucene50(blocksize=128), 
defaultExtr=FST50}, docValues:{}, maxPointsInLeafNode=900, 
maxMBSortInHeap=5.592868203506101, sim=RandomSimilarity(queryNorm=false): {}, 
locale=fr-BE, timezone=SystemV/PST8
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_121 (64-bit)/cpus=4,threads=1,free=100355944,total=264241152
   [junit4]   2> NOTE: All tests run in this JVM: [TestExtractionDateUtil, 
ExtractingRequestHandlerTest]
   [junit4] Completed [4/4 (1!)] on J1 in 13.19s, 16 tests, 1 error <<< 
FAILURES!

[...truncated 57347 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to