Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/183/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:132)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:177)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)
  at sun.reflect.GeneratedConstructorAccessor200.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:799)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:861)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1114)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:984)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:869)  at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1138)
  at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:684)  
at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
 at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.update.HdfsTransactionLog
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:132)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:177)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:119)
        at sun.reflect.GeneratedConstructorAccessor200.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:799)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:861)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1114)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:984)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:869)
        at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1138)
        at 
org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:684)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)


        at __randomizedtesting.SeedInfo.seed([BE5155F3D629A536]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:304)
        at sun.reflect.GeneratedMethodAccessor59.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.common.cloud.TestCollectionStateWatchers.testSimpleCollectionWatch

Error Message:
CollectionStateWatcher was never notified of cluster change

Stack Trace:
java.lang.AssertionError: CollectionStateWatcher was never notified of cluster 
change
        at 
__randomizedtesting.SeedInfo.seed([466EB2632B3B0B17:1B557D136C369429]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.common.cloud.TestCollectionStateWatchers.testSimpleCollectionWatch(TestCollectionStateWatchers.java:141)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14164 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/init-core-data-001
   [junit4]   2> 3441894 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-001
   [junit4]   2> 3441894 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3441895 INFO  (Thread-10315) [    ] o.a.s.c.ZkTestServer 
client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3441895 INFO  (Thread-10315) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 3441898 ERROR (Thread-10315) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3441995 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:41605
   [junit4]   2> 3441999 INFO  (zkConnectionManagerCallback-14881-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442005 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: 
d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 3442005 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: 
d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 3442007 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3442007 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3442007 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3442007 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3442007 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3442008 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1de4c7a2{/solr,null,AVAILABLE}
   [junit4]   2> 3442008 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3442008 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@24ccc5af{/solr,null,AVAILABLE}
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@985500e{SSL,[ssl, 
http/1.1]}{127.0.0.1:38068}
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@4c3d0cb9{SSL,[ssl, 
http/1.1]}{127.0.0.1:44148}
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.e.j.s.Server Started @3442086ms
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.e.j.s.Server Started @3442086ms
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=38068}
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=44148}
   [junit4]   2> 3442011 ERROR (jetty-launcher-14878-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3442011 ERROR (jetty-launcher-14878-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter Using logger factory 
org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.6.0
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
7.6.0
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2018-10-11T14:13:23.096Z
   [junit4]   2> 3442011 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2018-10-11T14:13:23.096Z
   [junit4]   2> 3442013 INFO  (zkConnectionManagerCallback-14885-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442013 INFO  (zkConnectionManagerCallback-14884-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442014 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3442014 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3442112 INFO  (jetty-launcher-14878-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41605/solr
   [junit4]   2> 3442113 INFO  (zkConnectionManagerCallback-14889-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442115 INFO  (zkConnectionManagerCallback-14891-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442155 INFO  (jetty-launcher-14878-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41605/solr
   [junit4]   2> 3442156 INFO  (zkConnectionManagerCallback-14897-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442159 INFO  (zkConnectionManagerCallback-14899-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442173 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:44148_solr
   [junit4]   2> 3442175 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.Overseer Overseer 
(id=72083938947694598-127.0.0.1:44148_solr-n_0000000000) starting
   [junit4]   2> 3442184 INFO  (zkConnectionManagerCallback-14906-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442186 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:41605/solr ready
   [junit4]   2> 3442187 INFO  
(OverseerStateUpdate-72083938947694598-127.0.0.1:44148_solr-n_0000000000) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.Overseer Starting to work on the main 
queue : 127.0.0.1:44148_solr
   [junit4]   2> 3442188 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:44148_solr
   [junit4]   2> 3442194 INFO  
(OverseerStateUpdate-72083938947694598-127.0.0.1:44148_solr-n_0000000000) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3442198 INFO  (zkCallback-14905-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3442202 DEBUG 
(OverseerAutoScalingTriggerThread-72083938947694598-127.0.0.1:44148_solr-n_0000000000)
 [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial 
livenodes: [127.0.0.1:44148_solr]
   [junit4]   2> 3442204 DEBUG (ScheduledTrigger-13021-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 3442223 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 3442229 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 3442235 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating 
transient cache for 2147483647 transient cores
   [junit4]   2> 3442236 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:38068_solr
   [junit4]   2> 3442237 INFO  (zkCallback-14898-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442243 INFO  (zkCallback-14905-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442243 INFO  (zkCallback-14890-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3442264 INFO  (zkConnectionManagerCallback-14912-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442265 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 3442265 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_44148.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442266 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster 
at 127.0.0.1:41605/solr ready
   [junit4]   2> 3442267 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system 
collection, keeping metrics history in memory.
   [junit4]   2> 3442284 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_44148.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442285 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_44148.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442287 INFO  (jetty-launcher-14878-thread-1) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-001/node1/.
   [junit4]   2> 3442295 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38068.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442310 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38068.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442310 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38068.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3442312 INFO  (jetty-launcher-14878-thread-2) 
[n:127.0.0.1:38068_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-001/node2/.
   [junit4]   2> 3442357 INFO  (zkConnectionManagerCallback-14915-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442361 INFO  (zkConnectionManagerCallback-14920-thread-1) [   
 ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3442363 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3442364 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41605/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 3442400 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 3442406 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3442408 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3442427 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_42774_hdfs____.hec4zk/webapp
   [junit4]   2> 3442822 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42774
   [junit4]   2> 3442879 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3442880 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3442898 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_36787_datanode____droo0q/webapp
   [junit4]   2> 3443205 DEBUG (ScheduledTrigger-13021-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3443299 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36787
   [junit4]   2> 3443345 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3443346 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 3443365 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_34950_datanode____.2hq4na/webapp
   [junit4]   2> 3443394 ERROR (DataNode: 
[[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-002/hdfsBaseDir/data/data1/,
 
[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-002/hdfsBaseDir/data/data2/]]
  heartbeating to localhost/127.0.0.1:38241) [    ] 
o.a.h.h.s.d.DirectoryScanner 
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 
ms/sec. Assuming default value of 1000
   [junit4]   2> 3443407 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x16d9c45dad256: from storage 
DS-e96e4314-3ed7-496d-a8d3-a68ec552d71a node 
DatanodeRegistration(127.0.0.1:40673, 
datanodeUuid=e376e980-3648-4d66-88e5-544d0bc1e463, infoPort=38436, 
infoSecurePort=0, ipcPort=45469, 
storageInfo=lv=-56;cid=testClusterID;nsid=1154496238;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3443407 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x16d9c45dad256: from storage 
DS-e871cb58-3952-4638-a627-72bebb58b818 node 
DatanodeRegistration(127.0.0.1:40673, 
datanodeUuid=e376e980-3648-4d66-88e5-544d0bc1e463, infoPort=38436, 
infoSecurePort=0, ipcPort=45469, 
storageInfo=lv=-56;cid=testClusterID;nsid=1154496238;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3443769 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[BE5155F3D629A536]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34950
   [junit4]   2> 3443847 ERROR (DataNode: 
[[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-002/hdfsBaseDir/data/data3/,
 
[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-002/hdfsBaseDir/data/data4/]]
  heartbeating to localhost/127.0.0.1:38241) [    ] 
o.a.h.h.s.d.DirectoryScanner 
dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 
ms/sec. Assuming default value of 1000
   [junit4]   2> 3443854 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x16d9c607913dd: from storage 
DS-8a9c0d12-377e-419a-9cc6-1dd1e93ad0a3 node 
DatanodeRegistration(127.0.0.1:44034, 
datanodeUuid=57542ff0-106a-4577-bcdb-ee24e3f648d7, infoPort=33619, 
infoSecurePort=0, ipcPort=40707, 
storageInfo=lv=-56;cid=testClusterID;nsid=1154496238;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 3443854 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x16d9c607913dd: from storage 
DS-e73972ed-549a-436c-90ca-b3b741a419c2 node 
DatanodeRegistration(127.0.0.1:44034, 
datanodeUuid=57542ff0-106a-4577-bcdb-ee24e3f648d7, infoPort=33619, 
infoSecurePort=0, ipcPort=40707, 
storageInfo=lv=-56;cid=testClusterID;nsid=1154496238;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 3443995 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 3443998 INFO  
(OverseerThreadFactory-13023-thread-1-processing-n:127.0.0.1:44148_solr) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 3444110 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics 
params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core}
 status=0 QTime=1
   [junit4]   2> 3444131 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics 
params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core}
 status=0 QTime=0
   [junit4]   2> 3444137 INFO  
(OverseerStateUpdate-72083938947694598-127.0.0.1:44148_solr-n_0000000000) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"recoverytest",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"recoverytest_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:44148/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3444140 INFO  
(OverseerStateUpdate-72083938947694598-127.0.0.1:44148_solr-n_0000000000) 
[n:127.0.0.1:44148_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"recoverytest",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"recoverytest_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:38068/solr";,
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3444206 DEBUG (ScheduledTrigger-13021-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3444354 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr    
x:recoverytest_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3444355 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr    
x:recoverytest_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create 
command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3444355 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr    
x:recoverytest_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault 
Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3445206 DEBUG (ScheduledTrigger-13021-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3445381 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
   [junit4]   2> 3445382 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
   [junit4]   2> 3445390 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.IndexSchema [recoverytest_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3445390 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.IndexSchema [recoverytest_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3445394 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3445394 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3445394 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using 
configuration from collection recoverytest, trusted=true
   [junit4]   2> 3445394 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using 
configuration from collection recoverytest, trusted=true
   [junit4]   2> 3445395 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_44148.solr.core.recoverytest.shard1.replica_n1' (registry 
'solr.core.recoverytest.shard1.replica_n1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3445395 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_38068.solr.core.recoverytest.shard1.replica_n2' (registry 
'solr.core.recoverytest.shard1.replica_n2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@2623b591
   [junit4]   2> 3445395 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:38241/data
   [junit4]   2> 3445395 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:38241/data
   [junit4]   2> 3445395 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 3445395 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 3445395 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.SolrCore [[recoverytest_shard1_replica_n2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-001/node2/recoverytest_shard1_replica_n2],
 dataDir=[hdfs://localhost:38241/data/recoverytest/core_node4/data/]
   [junit4]   2> 3445395 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.SolrCore [[recoverytest_shard1_replica_n1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_BE5155F3D629A536-001/tempDir-001/node1/recoverytest_shard1_replica_n1],
 dataDir=[hdfs://localhost:38241/data/recoverytest/core_node3/data/]
   [junit4]   2> 3445398 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node4/data/snapshot_metadata
   [junit4]   2> 3445398 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node3/data/snapshot_metadata
   [junit4]   2> 3445407 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3445407 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3445407 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3445407 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3445418 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3445419 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3445420 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node4/data
   [junit4]   2> 3445420 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node3/data
   [junit4]   2> 3445441 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node4/data/index
   [junit4]   2> 3445447 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:38241/data/recoverytest/core_node3/data/index
   [junit4]   2> 3445449 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3445449 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3445455 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 3445455 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 3445459 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3445461 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 3445489 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44034 is added to 
blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-e96e4314-3ed7-496d-a8d3-a68ec552d71a:NORMAL:127.0.0.1:40673|RBW],
 
ReplicaUC[[DISK]DS-8a9c0d12-377e-419a-9cc6-1dd1e93ad0a3:NORMAL:127.0.0.1:44034|RBW]]}
 size 0
   [junit4]   2> 3445490 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40673 is added to 
blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-e96e4314-3ed7-496d-a8d3-a68ec552d71a:NORMAL:127.0.0.1:40673|RBW],
 
ReplicaUC[[DISK]DS-e73972ed-549a-436c-90ca-b3b741a419c2:NORMAL:127.0.0.1:44034|RBW]]}
 size 0
   [junit4]   2> 3445492 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44034 is added to 
blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-e96e4314-3ed7-496d-a8d3-a68ec552d71a:NORMAL:127.0.0.1:40673|RBW],
 
ReplicaUC[[DISK]DS-e73972ed-549a-436c-90ca-b3b741a419c2:NORMAL:127.0.0.1:44034|RBW]]}
 size 0
   [junit4]   2> 3445492 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40673 is added to 
blk_1073741825_1001 size 69
   [junit4]   2> 3445583 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 3445583 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 3445583 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3445583 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3445583 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 3445583 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 3445597 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3445597 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3445597 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3445597 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3445611 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1456de65[recoverytest_shard1_replica_n1] main]
   [junit4]   2> 3445612 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@72157984[recoverytest_shard1_replica_n2] main]
   [junit4]   2> 3445613 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 3445613 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 3445613 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3445615 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3445616 WARN  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://localhost:40328/solr3 due to exception: java.net.ConnectException: Call 
From lucene1-us-west/10.41.0.5 to localhost:40328 failed on connection 
exception: java.net.ConnectException: Connection refused; For more details see: 
 http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 3445617 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3445617 INFO  
(searcherExecutor-13033-thread-1-processing-n:127.0.0.1:38068_solr 
x:recoverytest_shard1_replica_n2 c:recoverytest s:shard1 r:core_node4) 
[n:127.0.0.1:38068_solr c:recoverytest s:shard1 r:core_node4 
x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica_n2] Registered new searcher 
Searcher@72157984[recoverytest_shard1_replica_n2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3445617 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1614038650534756352
   [junit4]   2> 3445618 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3445619 WARN  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for 
hdfs://localhost:40328/solr3 due to exception: java.net.ConnectException: Call 
From lucene1-us-west/10.41.0.5 to localhost:40328 failed on connection 
exception: java.net.ConnectException: Connection refused; For more details see: 
 http://wiki.apache.org/hadoop/ConnectionRefused
   [junit4]   2> 3445619 INFO  
(searcherExecutor-13032-thread-1-processing-n:127.0.0.1:44148_solr 
x:recoverytest_shard1_replica_n1 c:recoverytest s:shard1 r:core_node3) 
[n:127.0.0.1:44148_solr c:recoverytest s:shard1 r:core_node3 
x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica_n1] Registered new searcher 
Searcher@1456de65[recoverytest_shard1_replica_n1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3445620 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1614038650537902080
   [junit4]   2> 3445625 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/recoverytest/terms/shard1 to Terms{values={core_node3=0}, 
version=0}
   [junit4]   2> 3445625 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.ZkShardTerms Failed to save terms, version is not a match, retrying
   [junit4]   2> 3445626 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/recoverytest/terms/shard1 to Terms{values={core_node3=0, 
core_node4=0}, version=1}
   [junit4]   2> 3445631 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3445631 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3445631 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/
   [junit4]   2> 3445632 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 
url=https://127.0.0.1:44148/solr START 
replicas=[https://127.0.0.1:38068/solr/recoverytest_shard1_replica_n2/] 
nUpdates=100
   [junit4]   2> 3445632 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 
url=https://127.0.0.1:44148/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 3445634 INFO  (qtp548887852-44299) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.c.S.Request [recoverytest_shard1_replica_n2]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=1
   [junit4]   2> 3445634 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the 
next candidate
   [junit4]   2> 3445634 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 3445635 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 3445639 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/ shard1
   [junit4]   2> 3445742 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3445746 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=1391
   [junit4]   2> 3445845 INFO  (zkCallback-14898-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3446206 DEBUG (ScheduledTrigger-13021-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3446635 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT}
 status=0 QTime=2281
   [junit4]   2> 3446638 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 3446735 INFO  (zkCallback-14898-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3446735 INFO  (zkCallback-14890-thread-1) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 3447207 DEBUG (ScheduledTrigger-13021-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 3447638 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=3643
   [junit4]   2> 3447640 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[BE5155F3D629A536]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 1350 documents
   [junit4]   2> 3447652 DEBUG (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://localhost:38241/data/recoverytest/core_node3/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 3447653 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Successful update of terms at 
/collections/recoverytest/terms/shard1 to Terms{values={core_node3=1, 
core_node4=1}, version=2}
   [junit4]   2> 3447653 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.c.ZkShardTerms Failed to save terms, version is not a match, retrying
   [junit4]   2> 3447657 DEBUG (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.HdfsTransactionLog Opening new tlog hdfs 
tlog{file=hdfs://localhost:38241/data/recoverytest/core_node4/data/tlog/tlog.0000000000000000000
 refcount=1}
   [junit4]   2> 3447673 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-0
 (1614038652662317056)]} 0 17
   [junit4]   2> 3447673 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1614038652662317056)]} 0 27
   [junit4]   2> 3447674 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-0
 (1614038652663365632)]} 0 19
   [junit4]   2> 3447674 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1614038652663365632)]} 0 27
   [junit4]   2> 3447677 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-1
 (1614038652691677184)]} 0 1
   [junit4]   2> 3447677 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1614038652691677184)]} 0 2
   [junit4]   2> 3447677 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-1
 (1614038652692725760)]} 0 0
   [junit4]   2> 3447677 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1614038652692725760)]} 0 2
   [junit4]   2> 3447679 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-2
 (1614038652695871488)]} 0 0
   [junit4]   2> 3447679 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1614038652695871488)]} 0 1
   [junit4]   2> 3447679 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-2
 (1614038652695871489)]} 0 0
   [junit4]   2> 3447680 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1614038652695871489)]} 0 1
   [junit4]   2> 3447682 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-3
 (1614038652697968641)]} 0 0
   [junit4]   2> 3447682 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-3
 (1614038652697968640)]} 0 1
   [junit4]   2> 3447682 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 
(1614038652697968641)]} 0 1
   [junit4]   2> 3447682 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1614038652697968640)]} 0 1
   [junit4]   2> 3447684 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-4
 (1614038652701114369)]} 0 1
   [junit4]   2> 3447684 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-4
 (1614038652701114368)]} 0 1
   [junit4]   2> 3447685 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 
(1614038652701114369)]} 0 2
   [junit4]   2> 3447685 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 
(1614038652701114368)]} 0 2
   [junit4]   2> 3447687 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-2
 (-1614038652704260096)]} 0 0
   [junit4]   2> 3447687 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-2
 (-1614038652704260097)]} 0 0
   [junit4]   2> 3447688 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 
(-1614038652704260096)]} 0 1
   [junit4]   2> 3447688 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 
(-1614038652704260097)]} 0 1
   [junit4]   2> 3447690 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-5
 (1614038652707405824)]} 0 0
   [junit4]   2> 3447690 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 
(1614038652707405824)]} 0 1
   [junit4]   2> 3447692 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-6
 (1614038652709502976)]} 0 0
   [junit4]   2> 3447692 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 
(1614038652709502976)]} 0 1
   [junit4]   2> 3447694 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-4
 (-1614038652711600128)]} 0 0
   [junit4]   2> 3447694 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 
(-1614038652711600128)]} 0 1
   [junit4]   2> 3447696 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-7
 (1614038652713697280)]} 0 0
   [junit4]   2> 3447696 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 
(1614038652713697280)]} 0 1
   [junit4]   2> 3447698 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-5
 (-1614038652715794432)]} 0 0
   [junit4]   2> 3447698 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 
(-1614038652715794432)]} 0 1
   [junit4]   2> 3447699 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-5
 (1614038652706357248)]} 0 10
   [junit4]   2> 3447699 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 
(1614038652706357248)]} 0 11
   [junit4]   2> 3447700 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-8
 (1614038652717891584)]} 0 0
   [junit4]   2> 3447700 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 
(1614038652717891584)]} 0 1
   [junit4]   2> 3447702 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-6
 (1614038652718940160)]} 0 1
   [junit4]   2> 3447702 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 
(1614038652718940160)]} 0 2
   [junit4]   2> 3447702 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-9
 (1614038652719988736)]} 0 0
   [junit4]   2> 3447703 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 
(1614038652719988736)]} 0 1
   [junit4]   2> 3447704 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-4
 (-1614038652722085888)]} 0 0
   [junit4]   2> 3447704 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 
(-1614038652722085888)]} 0 1
   [junit4]   2> 3447706 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-10
 (1614038652722085889)]} 0 1
   [junit4]   2> 3447706 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 
(1614038652722085889)]} 0 2
   [junit4]   2> 3447706 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-7
 (1614038652724183040)]} 0 1
   [junit4]   2> 3447707 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 
(1614038652724183040)]} 0 1
   [junit4]   2> 3447708 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-11
 (1614038652726280192)]} 0 0
   [junit4]   2> 3447708 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 
(1614038652726280192)]} 0 1
   [junit4]   2> 3447710 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-12
 (1614038652728377344)]} 0 0
   [junit4]   2> 3447710 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 
(1614038652728377344)]} 0 1
   [junit4]   2> 3447712 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-8
 (-1614038652730474496)]} 0 0
   [junit4]   2> 3447712 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 
(-1614038652730474496)]} 0 1
   [junit4]   2> 3447714 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-13
 (1614038652732571648)]} 0 0
   [junit4]   2> 3447714 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 
(1614038652732571648)]} 0 1
   [junit4]   2> 3447716 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-10
 (-1614038652734668800)]} 0 0
   [junit4]   2> 3447716 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 
(-1614038652734668800)]} 0 1
   [junit4]   2> 3447718 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-14
 (1614038652736765952)]} 0 0
   [junit4]   2> 3447719 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 
(1614038652736765952)]} 0 1
   [junit4]   2> 3447719 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-5
 (-1614038652726280193)]} 0 10
   [junit4]   2> 3447719 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 
(-1614038652726280193)]} 0 11
   [junit4]   2> 3447723 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-8
 (1614038652742008832)]} 0 0
   [junit4]   2> 3447723 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 
(1614038652742008832)]} 0 1
   [junit4]   2> 3447725 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-9
 (1614038652744105984)]} 0 0
   [junit4]   2> 3447725 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 
(1614038652744105984)]} 0 1
   [junit4]   2> 3447727 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-10
 (1614038652746203136)]} 0 0
   [junit4]   2> 3447728 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 
(1614038652746203136)]} 0 1
   [junit4]   2> 3447729 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-11
 (1614038652748300288)]} 0 0
   [junit4]   2> 3447730 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 
(1614038652748300288)]} 0 1
   [junit4]   2> 3447730 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-15
 (1614038652739911680)]} 0 10
   [junit4]   2> 3447731 INFO  (qtp479969946-44296) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 
(1614038652739911680)]} 0 11
   [junit4]   2> 3447732 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-12
 (1614038652750397440)]} 0 1
   [junit4]   2> 3447732 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 
(1614038652750397440)]} 0 2
   [junit4]   2> 3447734 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-8
 (-1614038652753543168)]} 0 0
   [junit4]   2> 3447734 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 
(-1614038652753543168)]} 0 1
   [junit4]   2> 3447736 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-13
 (1614038652755640320)]} 0 0
   [junit4]   2> 3447736 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 
(1614038652755640320)]} 0 1
   [junit4]   2> 3447739 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-10
 (-1614038652757737472)]} 0 0
   [junit4]   2> 3447739 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 
(-1614038652757737472)]} 0 1
   [junit4]   2> 3447741 INFO  (qtp548887852-44301) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-14
 (1614038652760883200)]} 0 0
   [junit4]   2> 3447741 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 
(1614038652760883200)]} 0 1
   [junit4]   2> 3447742 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-16
 (1614038652751446016)]} 0 10
   [junit4]   2> 3447743 INFO  (qtp479969946-44302) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 
(1614038652751446016)]} 0 11
   [junit4]   2> 3447743 INFO  (qtp548887852-44304) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-15
 (1614038652762980352)]} 0 0
   [junit4]   2> 3447744 INFO  (qtp479969946-44300) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 
(1614038652762980352)]} 0 1
   [junit4]   2> 3447746 INFO  (qtp548887852-44291) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:44148/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-16
 (1614038652765077504)]} 0 0
   [junit4]   2> 3447746 INFO  (qtp479969946-44298) [n:127.0.0.1:44148_solr 
c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 
(1614038652765077504)]} 0 1
   [junit4]   2> 3447748 INFO  (qtp548887852-44297) [n:127.0.0.1:38068_solr 
c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2]  
webapp=/sol

[...truncated too long message...]

gure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null306344949
     [copy] Copying 239 files to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null306344949
   [delete] Deleting directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/null306344949

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: 
org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath]         confs: [default]
[ivy:cachepath]         found 
org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath]         found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath]         found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath]         found org.apache.httpcomponents#httpclient;4.3.6 in 
public
[ivy:cachepath]         found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath]         found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath]         found commons-codec#commons-codec;1.6 in public
[ivy:cachepath]         found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 28ms :: artifacts dl 2ms
        ---------------------------------------------------------------------
        |                  |            modules            ||   artifacts   |
        |       conf       | number| search|dwnlded|evicted|| number|dwnlded|
        ---------------------------------------------------------------------
        |      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
        ---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for 
further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 126 minutes 54 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
        at hudson.FilePath$34.hasMatch(FilePath.java:2678)
        at hudson.FilePath$34.invoke(FilePath.java:2557)
        at hudson.FilePath$34.invoke(FilePath.java:2547)
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
                at 
hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
                at 
hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
                at hudson.remoting.Channel.call(Channel.java:955)
                at hudson.FilePath.act(FilePath.java:1036)
                at hudson.FilePath.act(FilePath.java:1025)
                at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
                at 
hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
                at 
hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
                at 
hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
                at 
hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
                at 
hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
                at hudson.model.Build$BuildExecution.post2(Build.java:186)
                at 
hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
                at hudson.model.Run.execute(Run.java:1819)
                at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
                at 
hudson.model.ResourceController.execute(ResourceController.java:97)
                at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
        at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
        at hudson.remoting.UserRequest.perform(UserRequest.java:212)
        at hudson.remoting.UserRequest.perform(UserRequest.java:54)
        at hudson.remoting.Request$2.run(Request.java:369)
        at 
hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no 
matches found within 10000
        at hudson.FilePath.act(FilePath.java:1038)
        at hudson.FilePath.act(FilePath.java:1025)
        at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
        at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
        at 
hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
        at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
        at 
hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
        at 
hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
        at hudson.model.Build$BuildExecution.post2(Build.java:186)
        at 
hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
        at hudson.model.Run.execute(Run.java:1819)
        at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
        at hudson.model.ResourceController.execute(ResourceController.java:97)
        at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern 
"**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to