Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/413/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseParallelGC

2 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at 
org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) 
Thread[id=22012, name=jetty-launcher-2972-thread-1-EventThread, 
state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)  
       at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)      
   at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
         at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
         at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
         at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
         at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
         at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)   
      at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)   
 2) Thread[id=22019, name=jetty-launcher-2972-thread-2-EventThread, 
state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)  
       at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)      
   at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
         at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
         at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
         at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
         at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
         at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)   
      at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=22012, name=jetty-launcher-2972-thread-1-EventThread, 
state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=22019, name=jetty-launcher-2972-thread-2-EventThread, 
state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
        at __randomizedtesting.SeedInfo.seed([E5FB6AB026C44FCC]:0)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
        at 
__randomizedtesting.SeedInfo.seed([E5FB6AB026C44FCC:668D3542F0BD416D]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:184)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11868 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
   [junit4]   2> 360162 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/init-core-data-001
   [junit4]   2> 360162 WARN  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=206 numCloses=206
   [junit4]   2> 360162 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 360164 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 360164 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001
   [junit4]   2> 360164 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 360165 INFO  (Thread-1057) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 360165 INFO  (Thread-1057) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 360166 ERROR (Thread-1057) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 360265 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:44649
   [junit4]   2> 360274 INFO  (jetty-launcher-702-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 360274 INFO  (jetty-launcher-702-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 360276 INFO  (jetty-launcher-702-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1aa5be9{/solr,null,AVAILABLE}
   [junit4]   2> 360276 INFO  (jetty-launcher-702-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@183d6a6{/solr,null,AVAILABLE}
   [junit4]   2> 360276 INFO  (jetty-launcher-702-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@13e9f68{SSL,[ssl, 
http/1.1]}{127.0.0.1:41459}
   [junit4]   2> 360276 INFO  (jetty-launcher-702-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@19c7c9b{SSL,[ssl, 
http/1.1]}{127.0.0.1:35795}
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.e.j.s.Server Started @361259ms
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.e.j.s.Server Started @361259ms
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=41459}
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=35795}
   [junit4]   2> 360277 ERROR (jetty-launcher-702-thread-2) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 360277 ERROR (jetty-launcher-702-thread-1) [    ] 
o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.1.0
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.1.0
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config 
dir: null
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-09-14T16:27:13.381Z
   [junit4]   2> 360277 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-09-14T16:27:13.381Z
   [junit4]   2> 360278 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 360278 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 360289 INFO  (jetty-launcher-702-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44649/solr
   [junit4]   2> 360289 INFO  (jetty-launcher-702-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44649/solr
   [junit4]   2> 360336 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 360336 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 360337 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:35795_solr
   [junit4]   2> 360337 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:41459_solr
   [junit4]   2> 360337 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.Overseer Overseer 
(id=98658316010782726-127.0.0.1:35795_solr-n_0000000000) starting
   [junit4]   2> 360338 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 360338 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 360350 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:35795_solr
   [junit4]   2> 360351 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 360351 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 360443 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_41459.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360453 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_41459.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360453 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_41459.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360454 INFO  (jetty-launcher-702-thread-2) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node2/.
   [junit4]   2> 360523 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35795.solr.node' (registry 'solr.node') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360534 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35795.solr.jvm' (registry 'solr.jvm') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360535 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35795.solr.jetty' (registry 'solr.jetty') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 360536 INFO  (jetty-launcher-702-thread-1) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node1/.
   [junit4]   2> 360563 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 360563 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44649/solr ready
   [junit4]   2> 360605 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 360606 INFO  
(OverseerThreadFactory-2203-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
collection1
   [junit4]   2> 360712 INFO  
(OverseerStateUpdate-98658316010782726-127.0.0.1:35795_solr-n_0000000000) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35795/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 360713 INFO  
(OverseerStateUpdate-98658316010782726-127.0.0.1:35795_solr-n_0000000000) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:41459/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 360714 INFO  
(OverseerStateUpdate-98658316010782726-127.0.0.1:35795_solr-n_0000000000) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35795/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 360714 INFO  
(OverseerStateUpdate-98658316010782726-127.0.0.1:35795_solr-n_0000000000) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_t6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:41459/solr";,
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 360925 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 360925 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 360926 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 360959 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 360959 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 360967 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 361070 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 361070 INFO  
(zkCallback-714-thread-2-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 361070 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 361070 INFO  
(zkCallback-713-thread-2-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 361938 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 361938 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 361951 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.s.IndexSchema [collection1_shard2_replica_t6] Schema name=minimal
   [junit4]   2> 361951 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.s.IndexSchema [collection1_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 361956 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 361956 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 361956 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t2' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 361956 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t6' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 361956 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_41459.solr.core.collection1.shard2.replica_t6' (registry 
'solr.core.collection1.shard2.replica_t6') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 361956 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_41459.solr.core.collection1.shard1.replica_t2' (registry 
'solr.core.collection1.shard1.replica_t2') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 361957 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 361957 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.SolrCore [[collection1_shard2_replica_t6] ] Opening new SolrCore at 
[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node2/collection1_shard2_replica_t6],
 
dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node2/./collection1_shard2_replica_t6/data/]
   [junit4]   2> 361957 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 361957 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.SolrCore [[collection1_shard1_replica_t2] ] Opening new SolrCore at 
[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node2/collection1_shard1_replica_t2],
 
dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node2/./collection1_shard1_replica_t2/data/]
   [junit4]   2> 361972 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 361975 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
   [junit4]   2> 361989 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 361989 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.s.IndexSchema [collection1_shard2_replica_t4] Schema name=minimal
   [junit4]   2> 361992 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 361992 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 361992 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 361992 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t4' using 
configuration from collection collection1, trusted=true
   [junit4]   2> 361992 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35795.solr.core.collection1.shard1.replica_t1' (registry 
'solr.core.collection1.shard1.replica_t1') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 361993 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.m.r.SolrJmxReporter JMX monitoring for 
'solr_35795.solr.core.collection1.shard2.replica_t4' (registry 
'solr.core.collection1.shard2.replica_t4') enabled at server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@1de04de
   [junit4]   2> 361993 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 361993 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 361993 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.SolrCore [[collection1_shard1_replica_t1] ] Opening new SolrCore at 
[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node1/collection1_shard1_replica_t1],
 
dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node1/./collection1_shard1_replica_t1/data/]
   [junit4]   2> 361993 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.SolrCore [[collection1_shard2_replica_t4] ] Opening new SolrCore at 
[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node1/collection1_shard2_replica_t4],
 
dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudRecovery_E5FB6AB026C44FCC-001/tempDir-001/node1/./collection1_shard2_replica_t4/data/]
   [junit4]   2> 362051 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 362051 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 362051 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 362051 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 362052 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 362052 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 362052 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 362052 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 362055 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@7b141f[collection1_shard2_replica_t6] main]
   [junit4]   2> 362055 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@1af721a[collection1_shard1_replica_t2] main]
   [junit4]   2> 362056 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 362056 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 362057 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 362058 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 362058 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 362058 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 362059 INFO  
(searcherExecutor-2208-thread-1-processing-n:127.0.0.1:41459_solr 
x:collection1_shard1_replica_t2 s:shard1 c:collection1 r:core_node5) 
[n:127.0.0.1:41459_solr c:collection1 s:shard1 r:core_node5 
x:collection1_shard1_replica_t2] o.a.s.c.SolrCore 
[collection1_shard1_replica_t2] Registered new searcher 
Searcher@1af721a[collection1_shard1_replica_t2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 362059 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1578533058157477888
   [junit4]   2> 362060 INFO  
(searcherExecutor-2209-thread-1-processing-n:127.0.0.1:41459_solr 
x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) 
[n:127.0.0.1:41459_solr c:collection1 s:shard2 r:core_node8 
x:collection1_shard2_replica_t6] o.a.s.c.SolrCore 
[collection1_shard2_replica_t6] Registered new searcher 
Searcher@7b141f[collection1_shard2_replica_t6] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 362060 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1578533058158526464
   [junit4]   2> 362064 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 362064 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 362071 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 362071 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 362072 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 362072 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 362074 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@457b0f[collection1_shard2_replica_t4] main]
   [junit4]   2> 362075 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 362076 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 362076 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 362077 INFO  
(searcherExecutor-2211-thread-1-processing-n:127.0.0.1:35795_solr 
x:collection1_shard2_replica_t4 s:shard2 c:collection1 r:core_node7) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.c.SolrCore 
[collection1_shard2_replica_t4] Registered new searcher 
Searcher@457b0f[collection1_shard2_replica_t4] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 362078 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1578533058177400832
   [junit4]   2> 362078 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 362078 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 362080 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 362080 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 362082 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@11f7b91[collection1_shard1_replica_t1] main]
   [junit4]   2> 362082 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/config
   [junit4]   2> 362083 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 362083 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 362084 INFO  
(searcherExecutor-2210-thread-1-processing-n:127.0.0.1:35795_solr 
x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.c.SolrCore 
[collection1_shard1_replica_t1] Registered new searcher 
Searcher@11f7b91[collection1_shard1_replica_t1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 362085 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1578533058184740864
   [junit4]   2> 362167 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362167 INFO  
(zkCallback-713-thread-2-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362167 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362167 INFO  
(zkCallback-714-thread-2-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362565 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 362565 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 362565 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 362565 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 362565 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/
   [junit4]   2> 362565 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:41459/solr/collection1_shard1_replica_t2/
   [junit4]   2> 362566 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t6 
url=https://127.0.0.1:41459/solr START 
replicas=[https://127.0.0.1:35795/solr/collection1_shard2_replica_t4/] 
nUpdates=100
   [junit4]   2> 362566 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 
url=https://127.0.0.1:41459/solr START 
replicas=[https://127.0.0.1:35795/solr/collection1_shard1_replica_t1/] 
nUpdates=100
   [junit4]   2> 362571 INFO  (qtp6849090-4282) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.S.Request [collection1_shard2_replica_t4]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 362571 INFO  (qtp6849090-4280) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.S.Request [collection1_shard1_replica_t1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 362866 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t6 
url=https://127.0.0.1:41459/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 362866 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 
url=https://127.0.0.1:41459/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 362866 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the 
next candidate
   [junit4]   2> 362866 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the 
next candidate
   [junit4]   2> 362866 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 362866 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 362866 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 362866 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 362867 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ZkController collection1_shard1_replica_t2 stopping background 
replication from leader
   [junit4]   2> 362867 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ZkController collection1_shard2_replica_t6 stopping background 
replication from leader
   [junit4]   2> 362869 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:41459/solr/collection1_shard1_replica_t2/ shard1
   [junit4]   2> 362869 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/ shard2
   [junit4]   2> 362972 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362972 INFO  
(zkCallback-714-thread-2-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362972 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 362972 INFO  
(zkCallback-713-thread-2-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 363020 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 363020 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 363021 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=2095
   [junit4]   2> 363021 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=2096
   [junit4]   2> 363084 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.ZkController collection1_shard2_replica_t4 starting background 
replication from leader
   [junit4]   2> 363084 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.c.ReplicateFromLeader Will start replication from leader with poll 
interval: 00:00:03
   [junit4]   2> 363086 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 363088 INFO  (qtp6849090-4201) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=2129
   [junit4]   2> 363088 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.ZkController collection1_shard1_replica_t1 starting background 
replication from leader
   [junit4]   2> 363088 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.c.ReplicateFromLeader Will start replication from leader with poll 
interval: 00:00:03
   [junit4]   2> 363090 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 363090 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Replica core_node5 is 
leader but it's state is down, skipping replication
   [junit4]   2> 363093 INFO  (qtp6849090-4202) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG}
 status=0 QTime=2126
   [junit4]   2> 363099 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 363198 INFO  
(zkCallback-713-thread-1-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 363198 INFO  
(zkCallback-713-thread-2-processing-n:127.0.0.1:41459_solr) 
[n:127.0.0.1:41459_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 363198 INFO  
(zkCallback-714-thread-2-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 363198 INFO  
(zkCallback-714-thread-1-processing-n:127.0.0.1:35795_solr) 
[n:127.0.0.1:35795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 364099 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2}
 status=0 QTime=3495
   [junit4]   2> 364100 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: 
collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 364101 INFO  
(SUITE-TestCloudRecovery-seed#[E5FB6AB026C44FCC]-worker) [    ] 
o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 364118 INFO  
(TEST-TestCloudRecovery.leaderRecoverFromLogOnStartupTest-seed#[E5FB6AB026C44FCC])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting leaderRecoverFromLogOnStartupTest
   [junit4]   2> 364125 INFO  (qtp6849090-4203) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&_version_=-1578533060318593024&distrib.from=https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/&wt=javabin&version=2}{deleteByQuery=*:*
 (-1578533060318593024)} 0 1
   [junit4]   2> 364128 INFO  (qtp6849090-4205) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&_version_=-1578533060322787328&distrib.from=https://127.0.0.1:41459/solr/collection1_shard1_replica_t2/&wt=javabin&version=2}{deleteByQuery=*:*
 (-1578533060322787328)} 0 0
   [junit4]   2> 364128 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2]  
webapp=/solr path=/update 
params={update.distrib=TOLEADER&distrib.from=https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/&wt=javabin&version=2}{deleteByQuery=*:*
 (-1578533060322787328)} 0 4
   [junit4]   2> 364129 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t6]  
webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* 
(-1578533060318593024)} 0 9
   [junit4]   2> 364133 INFO  (qtp6849090-4283) [n:127.0.0.1:35795_solr 
c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] 
o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 364133 INFO  (qtp6849090-4281) [n:127.0.0.1:35795_solr 
c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] 
o.a.s.u.TestInjection Start waiting for replica in sync with leader
   [junit4]   2> 364137 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1578533060336418816,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 364137 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 364137 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 364137 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 0
   [junit4]   2> 364138 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.DirectUpdateHandler2 start 
commit{_version_=1578533060337467392,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 364139 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 364139 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 364139 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t6]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 0
   [junit4]   2> 364139 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 364141 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 364608 INFO  
(OverseerCollectionConfigSetProcessor-98658316010782726-127.0.0.1:35795_solr-n_0000000000)
 [n:127.0.0.1:35795_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 364644 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 364646 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 365150 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 365151 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 365160 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Updated masterUrl to 
https://127.0.0.1:41459/solr/collection1_shard2_replica_t6/
   [junit4]   2> 365163 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 365163 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 365163 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 365164 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 365164 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 365657 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 365657 INFO  (qtp4071773-4266) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 366090 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Last replication failed, 
so I'll force replication
   [junit4]   2> 366090 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Updated masterUrl to 
https://127.0.0.1:41459/solr/collection1_shard1_replica_t2/
   [junit4]   2> 366092 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. 
Deleting mine...
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 366092 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.u.SolrIndexWriter Calling setCommitData 
with IW:org.apache.solr.update.SolrIndexWriter@11ccbe commitCommandVersion:0
   [junit4]   2> 366109 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening 
[Searcher@e9a7ce[collection1_shard1_replica_t1] main]
   [junit4]   2> 366110 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 366111 INFO  
(searcherExecutor-2210-thread-1-processing-n:127.0.0.1:35795_solr 
x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.c.SolrCore 
[collection1_shard1_replica_t1] Registered new searcher 
Searcher@e9a7ce[collection1_shard1_replica_t1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 366162 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 366163 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 366670 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 366670 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 367403 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 367403 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 367908 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 367908 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 368161 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 368162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 368162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 368162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 368162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 368413 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 368413 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 368917 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 368918 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 369090 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 369091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 369091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 369091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 369091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 
1505406439196
   [junit4]   2> 369422 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 369422 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 369926 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 369927 INFO  (qtp4071773-4266) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 370431 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 370433 INFO  (qtp4071773-4266) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 370937 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 370937 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 371161 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 371162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 371162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 371162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 371162 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 371441 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 371443 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 371946 INFO  (qtp4071773-4266) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 371948 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 372090 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 372091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 372091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 372091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 372091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 
1505406439196
   [junit4]   2> 372450 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 372452 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 372954 INFO  (qtp4071773-4216) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 372956 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 373459 INFO  (qtp4071773-4210) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 373460 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 373963 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 373964 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 374161 INFO  (qtp4071773-4266) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 374161 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 374161 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 374161 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 374161 INFO  (indexFetcher-2229-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard2 r:core_node7 
x:collection1_shard2_replica_t4] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 374467 INFO  (qtp4071773-4264) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 374469 INFO  (qtp4071773-4263) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 374972 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 374973 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 375090 INFO  (qtp4071773-4214) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 375091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 375091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 375091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 375091 INFO  (indexFetcher-2231-thread-1) 
[n:127.0.0.1:35795_solr c:collection1 s:shard1 r:core_node3 
x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 
1505406439196
   [junit4]   2> 375478 INFO  (qtp4071773-4204) [n:127.0.0.1:41459_solr 
c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] 
o.a.s.c.S.Request [collection1_shard1_replica_t2]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 375478 INFO  (qtp4071773-4208) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] 
o.a.s.c.S.Request [collection1_shard2_replica_t6]  webapp=/solr 
path=/replication params={qt=/replication&wt=javabin&version=2&command=details} 
status=0 QTime=0
   [junit4]   2> 375983 INFO  (qtp4071773-4215) [n:127.0.0.1:41459_solr 
c:collection1 s:shard2 r:core_n

[...truncated too long message...]

it4]   2>       at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2>        at 
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2>        at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1323, maxMBSortInHeap=6.914983600878435, 
sim=RandomSimilarity(queryNorm=true): {}, locale=el-GR, 
timezone=Antarctica/Vostok
   [junit4]   2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 
1.8.0_144 (32-bit)/cpus=8,threads=3,free=254731336,total=528744448
   [junit4]   2> NOTE: All tests run in this JVM: [TestCloudInspectUtil, 
CSVRequestHandlerTest, TestRebalanceLeaders, 
PeerSyncWithIndexFingerprintCachingTest, SliceStateTest, 
TestAtomicUpdateErrorCases, TestSearchPerf, TestExceedMaxTermLength, 
ParsingFieldUpdateProcessorsTest, SecurityConfHandlerTest, DateRangeFieldTest, 
AnalyticsQueryTest, TestStressInPlaceUpdates, TestTestInjection, 
TestOnReconnectListenerSupport, DOMUtilTest, TestSerializedLuceneMatchVersion, 
TermVectorComponentTest, DeleteInactiveReplicaTest, TestMacroExpander, 
SuggestComponentTest, SimpleCollectionCreateDeleteTest, TestInitQParser, 
CheckHdfsIndexTest, TestValueSourceCache, TestStandardQParsers, 
TestObjectReleaseTracker, IndexSchemaRuntimeFieldTest, 
TestPKIAuthenticationPlugin, ResourceLoaderTest, BasicAuthStandaloneTest, 
SchemaVersionSpecificBehaviorTest, TestQueryTypes, TestShortCircuitedRequests, 
TestJsonFacets, TestSolrCloudWithKerberosAlt, TestLocalFSCloudBackupRestore, 
TestFieldCacheWithThreads, TemplateUpdateProcessorTest, ZkCLITest, 
DocValuesTest, TestSolrJ, TestUpdate, ConjunctionSolrSpellCheckerTest, 
CdcrReplicationHandlerTest, TestSystemIdResolver, TestInPlaceUpdatesStandalone, 
OverseerRolesTest, TestDownShardTolerantSearch, SolrCloudReportersTest, 
CoreAdminHandlerTest, JavabinLoaderTest, AnalysisAfterCoreReloadTest, 
ConnectionReuseTest, OpenCloseCoreStressTest, TestComplexPhraseQParserPlugin, 
DocValuesMultiTest, DistributedFacetPivotSmallAdvancedTest, 
TestPullReplicaErrorHandling, TestFiltering, SoftAutoCommitTest, 
DistributedMLTComponentTest, MoreLikeThisHandlerTest, 
TestCursorMarkWithoutUniqueKey, LeaderInitiatedRecoveryOnShardRestartTest, 
TestNonDefinedSimilarityFactory, ClusterStateUpdateTest, MergeStrategyTest, 
DistributedIntervalFacetingTest, TestClusterProperties, TestDefaultStatsCache, 
DistributedTermsComponentTest, TestCoreContainer, SOLR749Test, 
ShardRoutingTest, TestPartialUpdateDeduplication, TestQueryWrapperFilter, 
RequestLoggingTest, TestLegacyNumericUtils, TestConfigOverlay, BufferStoreTest, 
TestReqParamsAPI, TestApiFramework, TestCoreDiscovery, TestSolr4Spatial2, 
CacheHeaderTest, TestCoreAdminApis, SpellCheckComponentTest, 
TestDynamicLoading, TestLRUCache, JsonLoaderTest, 
ShufflingReplicaListTransformerTest, TestInitParams, TestSearcherReuse, 
ForceLeaderTest, IndexSchemaTest, ZkSolrClientTest, DistributedQueueTest, 
DistribCursorPagingTest, TestBM25SimilarityFactory, HdfsBasicDistributedZkTest, 
TestSolrXml, RollingRestartTest, TestRequestStatusCollectionAPI, 
TestExactStatsCache, FacetPivotSmallTest, 
DistributedQueryComponentCustomSortTest, TestScoreJoinQPScore, 
TestHttpShardHandlerFactory, QueryParsingTest, InfoHandlerTest, 
TestConfigSetsAPI, TestElisionMultitermQuery, TestRTGBase, TestCryptoKeys, 
SimpleMLTQParserTest, TestZkAclsWithHadoopAuth, TestCharFilters, 
HdfsDirectoryFactoryTest, TestSolrIndexConfig, SuggesterTSTTest, 
ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, 
LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, 
TestReplicationHandler, ShardRoutingCustomTest, TestDistributedGrouping, 
ZkControllerTest, TestStressVersions, SolrCmdDistributorTest, HighlighterTest, 
SpellCheckCollatorTest, TestStressLucene, TestTrie, SuggesterWFSTTest, 
NoCacheHeaderTest, TestPseudoReturnFields, 
StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, 
DebugComponentTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, 
TestWriterPerf, DirectSolrSpellCheckerTest, TermsComponentTest, 
LoggingHandlerTest, ReturnFieldsTest, TestCSVResponseWriter, 
TestPHPSerializedResponseWriter, TestComponentsName, SearchHandlerTest, 
TestStressRecovery, PrimUtilsTest, TestSuggestSpellingConverter, 
RAMDirectoryFactoryTest, TestUtils, TestDocumentBuilder, ZkNodePropsTest, 
SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, TestCrossCoreJoin, 
TestDistributedMissingSort, TestSimpleTrackingShardHandler, 
TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, 
TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, 
AssignBackwardCompatibilityTest, AssignTest, BaseCdcrDistributedZkTest, 
CdcrRequestHandlerTest, CdcrVersionReplicationTest, 
ChaosMonkeySafeLeaderWithPullReplicasTest, CleanupOldIndexTest, 
CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, 
CollectionsAPISolrJTest, DistribDocExpirationUpdateProcessorTest, 
HttpPartitionTest, MoveReplicaHDFSTest, MoveReplicaTest, NodeMutatorTest, 
OutOfBoxZkACLAndCredentialsProvidersTest, 
OverriddenZkACLAndCredentialsProvidersTest, OverseerTaskQueueTest, 
PeerSyncReplicationTest, ReplaceNodeTest, RestartWhileUpdatingTest, 
ShardSplitTest, TestSizeLimitedDistributedMap, 
TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=E5FB6AB026C44FCC 
-Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=el-GR 
-Dtests.timezone=Antarctica/Vostok -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=22012, 
name=jetty-launcher-2972-thread-1-EventThread, state=TIMED_WAITING, 
group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at 
java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at 
org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=22019, 
name=jetty-launcher-2972-thread-2-EventThread, state=TIMED_WAITING, 
group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at 
java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at 
org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at 
org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at 
org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at 
org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at 
org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E5FB6AB026C44FCC]:0)
   [junit4] Completed [541/731 (2!)] on J1 in 52.74s, 12 tests, 1 error <<< 
FAILURES!

[...truncated 42767 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to