Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/119/ Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation Error Message: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 2) Thread[id=33572, name=jetty-launcher-5459-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 3) Thread[id=33648, name=jetty-launcher-5459-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 2) Thread[id=33572, name=jetty-launcher-5459-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 3) Thread[id=33648, name=jetty-launcher-5459-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) at __randomizedtesting.SeedInfo.seed([5931498DB2FCC92F]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) at __randomizedtesting.SeedInfo.seed([5931498DB2FCC92F]:0) FAILED: org.apache.solr.cloud.TestPullReplica.testKillLeader Error Message: Replica state not updated in cluster state null Live Nodes: [127.0.0.1:63468_solr, 127.0.0.1:38429_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ "pullReplicas":"1", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node3":{ "core":"pull_replica_test_kill_leader_shard1_replica_n1", "base_url":"http://127.0.0.1:63468/solr", "node_name":"127.0.0.1:63468_solr", "state":"down", "type":"NRT", "leader":"true"}, "core_node4":{ "core":"pull_replica_test_kill_leader_shard1_replica_p2", "base_url":"http://127.0.0.1:38429/solr", "node_name":"127.0.0.1:38429_solr", "state":"active", "type":"PULL"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"100", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"} Stack Trace: java.lang.AssertionError: Replica state not updated in cluster state null Live Nodes: [127.0.0.1:63468_solr, 127.0.0.1:38429_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ "pullReplicas":"1", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node3":{ "core":"pull_replica_test_kill_leader_shard1_replica_n1", "base_url":"http://127.0.0.1:63468/solr", "node_name":"127.0.0.1:63468_solr", "state":"down", "type":"NRT", "leader":"true"}, "core_node4":{ "core":"pull_replica_test_kill_leader_shard1_replica_p2", "base_url":"http://127.0.0.1:38429/solr", "node_name":"127.0.0.1:38429_solr", "state":"active", "type":"PULL"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"100", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"} at __randomizedtesting.SeedInfo.seed([5931498DB2FCC92F:1027BD39D0475D79]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401) at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12468 lines...] [junit4] Suite: org.apache.solr.cloud.TestPullReplica [junit4] 2> 2738010 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/init-core-data-001 [junit4] 2> 2738011 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 2738012 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) [junit4] 2> 2738012 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001 [junit4] 2> 2738012 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 2738012 INFO (Thread-8096) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 2738012 INFO (Thread-8096) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 2738013 ERROR (Thread-8096) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 2738112 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:57935 [junit4] 2> 2738127 INFO (jetty-launcher-5389-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 2738127 INFO (jetty-launcher-5389-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 2738129 INFO (jetty-launcher-5389-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2f015f13{/solr,null,AVAILABLE} [junit4] 2> 2738129 INFO (jetty-launcher-5389-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1fec2bac{HTTP/1.1,[http/1.1]}{127.0.0.1:38429} [junit4] 2> 2738129 INFO (jetty-launcher-5389-thread-1) [ ] o.e.j.s.Server Started @2740971ms [junit4] 2> 2738129 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38429} [junit4] 2> 2738129 ERROR (jetty-launcher-5389-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2738130 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.1.0 [junit4] 2> 2738130 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2738130 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null [junit4] 2> 2738130 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-18T14:14:48.636Z [junit4] 2> 2738135 INFO (jetty-launcher-5389-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@101172c4{/solr,null,AVAILABLE} [junit4] 2> 2738135 INFO (jetty-launcher-5389-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3d084c58{HTTP/1.1,[http/1.1]}{127.0.0.1:63468} [junit4] 2> 2738135 INFO (jetty-launcher-5389-thread-2) [ ] o.e.j.s.Server Started @2740977ms [junit4] 2> 2738135 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63468} [junit4] 2> 2738135 ERROR (jetty-launcher-5389-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2738136 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.1.0 [junit4] 2> 2738136 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2738136 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null [junit4] 2> 2738136 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-18T14:14:48.642Z [junit4] 2> 2738138 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 2738138 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 2738144 INFO (jetty-launcher-5389-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57935/solr [junit4] 2> 2738144 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df5b249c70001, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 2738145 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df5b249c70002, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 2738145 INFO (jetty-launcher-5389-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57935/solr [junit4] 2> 2738281 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 2738282 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38429_solr [junit4] 2> 2738284 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.c.Overseer Overseer (id=98504912962781189-127.0.0.1:38429_solr-n_0000000000) starting [junit4] 2> 2738298 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38429_solr [junit4] 2> 2738301 INFO (zkCallback-5400-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 2738315 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 2738321 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 2738326 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63468_solr [junit4] 2> 2738328 INFO (zkCallback-5401-thread-1-processing-n:127.0.0.1:63468_solr) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 2738333 INFO (zkCallback-5400-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 2738462 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738463 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738470 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738470 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738470 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738470 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2738471 INFO (jetty-launcher-5389-thread-1) [n:127.0.0.1:38429_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/. [junit4] 2> 2738471 INFO (jetty-launcher-5389-thread-2) [n:127.0.0.1:63468_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/. [junit4] 2> 2738506 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 2738508 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57935/solr ready [junit4] 2> 2738512 INFO (SUITE-TestPullReplica-seed#[5931498DB2FCC92F]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false [junit4] 2> 2738513 INFO (qtp1757047768-32912) [n:127.0.0.1:63468_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 2738514 INFO (qtp1757047768-32912) [n:127.0.0.1:63468_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1 [junit4] 2> 2738623 INFO (TEST-TestPullReplica.testCreateDelete-seed#[5931498DB2FCC92F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete { seed=[5931498DB2FCC92F:4221B8C1440C311F]} [junit4] 2> 2738625 INFO (qtp1274872603-32900) [n:127.0.0.1:38429_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=3&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_create_delete&action=CREATE&numShards=2 and sendToOCPQueue=true [junit4] 2> 2738629 INFO (OverseerThreadFactory-12717-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_create_delete [junit4] 2> 2738630 WARN (OverseerThreadFactory-12717-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance. [junit4] 2> 2738742 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:63468/solr", [junit4] 2> "type":"NRT"} [junit4] 2> 2738746 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:38429/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738755 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p4", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:63468/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738759 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_create_delete_shard1_replica_p6", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:38429/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738763 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_n8", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:63468/solr", [junit4] 2> "type":"NRT"} [junit4] 2> 2738768 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p10", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:38429/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738771 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p12", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:63468/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738774 INFO (OverseerStateUpdate-98504912962781189-127.0.0.1:38429_solr-n_0000000000) [n:127.0.0.1:38429_solr ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_create_delete", [junit4] 2> "shard":"shard2", [junit4] 2> "core":"pull_replica_test_create_delete_shard2_replica_p14", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:38429/solr", [junit4] 2> "type":"PULL"} [junit4] 2> 2738977 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2738977 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2738977 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 2738977 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 2738977 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 2738978 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p10&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2738986 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2738986 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p14&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2738986 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n8&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT [junit4] 2> 2738987 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p12&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL [junit4] 2> 2739097 INFO (zkCallback-5400-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2739098 INFO (zkCallback-5401-thread-1-processing-n:127.0.0.1:63468_solr) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2739098 INFO (zkCallback-5400-thread-2-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2739098 INFO (zkCallback-5401-thread-2-processing-n:127.0.0.1:63468_solr) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2739996 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2739997 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740002 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740003 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740017 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p6] Schema name=minimal [junit4] 2> 2740017 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_n1] Schema name=minimal [junit4] 2> 2740018 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p4] Schema name=minimal [junit4] 2> 2740019 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740019 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p6' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740019 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.core.pull_replica_test_create_delete.shard1.replica_p6' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740020 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740020 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p6/data/] [junit4] 2> 2740020 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740020 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p4' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740020 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.core.pull_replica_test_create_delete.shard1.replica_p4' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740020 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740019 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740020 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_n1' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740020 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_p4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_p4/data/] [junit4] 2> 2740021 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.core.pull_replica_test_create_delete.shard1.replica_n1' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740021 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740021 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_n1/data/] [junit4] 2> 2740028 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p10] Schema name=minimal [junit4] 2> 2740029 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740029 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p10' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740030 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.core.pull_replica_test_create_delete.shard2.replica_p10' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740030 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740030 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p10] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p10], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p10/data/] [junit4] 2> 2740031 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740047 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740047 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740068 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_n8] Schema name=minimal [junit4] 2> 2740068 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p2] Schema name=minimal [junit4] 2> 2740069 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740069 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_n8' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740069 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740069 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.core.pull_replica_test_create_delete.shard2.replica_n8' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740069 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740069 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p2' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740069 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_n8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_n8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_n8/data/] [junit4] 2> 2740069 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.core.pull_replica_test_create_delete.shard1.replica_p2' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740069 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740069 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p2/data/] [junit4] 2> 2740083 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p12] Schema name=minimal [junit4] 2> 2740084 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740084 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p12' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740085 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63468.solr.core.pull_replica_test_create_delete.shard2.replica_p12' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740085 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740085 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p12] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_p12], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_p12/data/] [junit4] 2> 2740085 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0 [junit4] 2> 2740109 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p14] Schema name=minimal [junit4] 2> 2740111 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 2740111 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p14' using configuration from collection pull_replica_test_create_delete, trusted=true [junit4] 2> 2740111 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38429.solr.core.pull_replica_test_create_delete.shard2.replica_p14' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b43910f [junit4] 2> 2740111 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 2740112 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p14] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p14], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_5931498DB2FCC92F-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p14/data/] [junit4] 2> 2740131 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740132 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740133 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.SolrIndexSearcher Opening [Searcher@32fb9ad2[pull_replica_test_create_delete_shard2_replica_p10] main] [junit4] 2> 2740150 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740151 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740153 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740154 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p10 starting background replication from leader [junit4] 2> 2740154 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740155 INFO (qtp1274872603-32902) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740176 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog [junit4] 2> 2740176 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 2740178 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740178 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740179 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f59a076[pull_replica_test_create_delete_shard1_replica_n1] main] [junit4] 2> 2740180 INFO (searcherExecutor-12725-thread-1-processing-n:127.0.0.1:38429_solr x:pull_replica_test_create_delete_shard2_replica_p10 s:shard2 c:pull_replica_test_create_delete r:core_node13) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p10] Registered new searcher Searcher@32fb9ad2[pull_replica_test_create_delete_shard2_replica_p10] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740182 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740182 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740183 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740183 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576078609679908864 [junit4] 2> 2740184 INFO (searcherExecutor-12724-thread-1-processing-n:127.0.0.1:63468_solr x:pull_replica_test_create_delete_shard1_replica_n1 s:shard1 c:pull_replica_test_create_delete r:core_node3) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_n1] Registered new searcher Searcher@6f59a076[pull_replica_test_create_delete_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740196 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740196 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740197 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@74418900[pull_replica_test_create_delete_shard1_replica_p4] main] [junit4] 2> 2740199 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. [junit4] 2> 2740199 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync [junit4] 2> 2740199 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard1_replica_n1/ [junit4] 2> 2740199 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me [junit4] 2> 2740199 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740199 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard1_replica_n1/ has no replicas [junit4] 2> 2740200 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR [junit4] 2> 2740200 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740204 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740204 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog [junit4] 2> 2740205 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 2740205 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p4 starting background replication from leader [junit4] 2> 2740205 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard1_replica_n1/ shard1 [junit4] 2> 2740205 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740206 INFO (searcherExecutor-12723-thread-1-processing-n:127.0.0.1:63468_solr x:pull_replica_test_create_delete_shard1_replica_p4 s:shard1 c:pull_replica_test_create_delete r:core_node7) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p4] Registered new searcher Searcher@74418900[pull_replica_test_create_delete_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740207 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740208 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740208 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740209 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d8c757d[pull_replica_test_create_delete_shard2_replica_n8] main] [junit4] 2> 2740211 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740211 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740212 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740212 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740212 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740212 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576078609710317568 [junit4] 2> 2740213 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@8b587a2[pull_replica_test_create_delete_shard1_replica_p6] main] [junit4] 2> 2740213 INFO (searcherExecutor-12726-thread-1-processing-n:127.0.0.1:63468_solr x:pull_replica_test_create_delete_shard2_replica_n8 s:shard2 c:pull_replica_test_create_delete r:core_node11) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_n8] Registered new searcher Searcher@5d8c757d[pull_replica_test_create_delete_shard2_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740218 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740222 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. [junit4] 2> 2740222 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync [junit4] 2> 2740222 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard2_replica_n8/ [junit4] 2> 2740222 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me [junit4] 2> 2740222 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.SyncStrategy http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard2_replica_n8/ has no replicas [junit4] 2> 2740222 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740224 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740223 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740225 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740226 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p6 starting background replication from leader [junit4] 2> 2740226 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740227 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740223 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR [junit4] 2> 2740230 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.SolrIndexSearcher Opening [Searcher@57c01515[pull_replica_test_create_delete_shard2_replica_p12] main] [junit4] 2> 2740232 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740232 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740232 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740233 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p12 starting background replication from leader [junit4] 2> 2740233 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740234 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740238 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63468/solr/pull_replica_test_create_delete_shard2_replica_n8/ shard2 [junit4] 2> 2740242 INFO (searcherExecutor-12728-thread-1-processing-n:127.0.0.1:63468_solr x:pull_replica_test_create_delete_shard2_replica_p12 s:shard2 c:pull_replica_test_create_delete r:core_node15) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p12] Registered new searcher Searcher@57c01515[pull_replica_test_create_delete_shard2_replica_p12] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740242 INFO (searcherExecutor-12722-thread-1-processing-n:127.0.0.1:38429_solr x:pull_replica_test_create_delete_shard1_replica_p6 s:shard1 c:pull_replica_test_create_delete r:core_node9) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p6] Registered new searcher Searcher@8b587a2[pull_replica_test_create_delete_shard1_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740245 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740245 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740246 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4fc5893f[pull_replica_test_create_delete_shard1_replica_p2] main] [junit4] 2> 2740248 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740249 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740249 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740250 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p2 starting background replication from leader [junit4] 2> 2740250 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740250 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 2740250 INFO (searcherExecutor-12727-thread-1-processing-n:127.0.0.1:38429_solr x:pull_replica_test_create_delete_shard1_replica_p2 s:shard1 c:pull_replica_test_create_delete r:core_node5) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p2] Registered new searcher Searcher@4fc5893f[pull_replica_test_create_delete_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740250 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 2740251 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740252 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.SolrIndexSearcher Opening [Searcher@3555c6d9[pull_replica_test_create_delete_shard2_replica_p14] main] [junit4] 2> 2740257 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 2740258 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 2740259 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 [junit4] 2> 2740259 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p14 starting background replication from leader [junit4] 2> 2740259 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 2740260 INFO (searcherExecutor-12730-thread-1-processing-n:127.0.0.1:38429_solr x:pull_replica_test_create_delete_shard2_replica_p14 s:shard2 c:pull_replica_test_create_delete r:core_node16) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p14] Registered new searcher Searcher@3555c6d9[pull_replica_test_create_delete_shard2_replica_p14] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 2740260 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 2740339 INFO (zkCallback-5400-thread-2-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2740339 INFO (zkCallback-5400-thread-1-processing-n:127.0.0.1:38429_solr) [n:127.0.0.1:38429_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2740339 INFO (zkCallback-5401-thread-1-processing-n:127.0.0.1:63468_solr) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2740339 INFO (zkCallback-5401-thread-2-processing-n:127.0.0.1:63468_solr) [n:127.0.0.1:63468_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2]) [junit4] 2> 2740354 INFO (qtp1274872603-32903) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node5 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1369 [junit4] 2> 2740357 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary [junit4] 2> 2740358 INFO (indexFetcher-12747-thread-1) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication [junit4] 2> 2740359 INFO (qtp1757047768-32914) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node3 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1381 [junit4] 2> 2740359 INFO (indexFetcher-12736-thread-1) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication [junit4] 2> 2740360 INFO (qtp1757047768-32911) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard1 r:core_node7 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1383 [junit4] 2> 2740364 INFO (qtp1274872603-32904) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p14] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p14&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1377 [junit4] 2> 2740376 INFO (indexFetcher-12756-thread-1) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication [junit4] 2> 2740379 INFO (qtp1274872603-32901) [n:127.0.0.1:38429_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1402 [junit4] 2> 2740388 INFO (indexFetcher-12759-thread-1) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication [junit4] 2> 2740388 INFO (qtp1757047768-32916) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p12] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p12&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL} status=0 QTime=1401 [junit4] 2> 2740389 INFO (qtp1757047768-32913) [n:127.0.0.1:63468_solr c:pull_replica_test_create_delete s:shard2 r:core_node11 x:pull_replica_test_create_delete_shard2_replica_n8] o.a.s.c.ZkController I am the leader, no recovery necessary [junit4] 2> [...truncated too long message...] lrCLIZkUtilsTest, TestRandomFaceting, TestDownShardTolerantSearch, OpenExchangeRatesOrgProviderTest, QueryEqualityTest, TestDefaultStatsCache, SynonymTokenizerTest, UpdateParamsTest, SuggesterFSTTest, TestPayloadScoreQParserPlugin, TestQueryUtils, OpenCloseCoreStressTest, DistributedQueryElevationComponentTest, TestDocTermOrds, TestCollationFieldDocValues, LukeRequestHandlerTest, TestLegacyField, DocumentAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestStressUserVersions, TestXmlQParser, TestRawTransformer, TestStressCloudBlindAtomicUpdates, SubstringBytesRefFilterTest, TestDynamicLoading, QueryResultKeyTest, TestFileDictionaryLookup, TestConfigsApi, PropertiesRequestHandlerTest, RuleEngineTest, TestFieldCacheSortRandom, LeaderInitiatedRecoveryOnShardRestartTest, TestHdfsCloudBackupRestore, CollectionsAPIDistributedZkTest, SolrGraphiteReporterTest, TestCharFilters, TestCSVLoader, BadCopyFieldTest, TestBM25SimilarityFactory, TestBadConfig, TestJsonRequest, HdfsRecoveryZkTest, TestExactSharedStatsCache, DistributedSuggestComponentTest, HdfsBasicDistributedZkTest, TestSortByMinMaxFunction, TermVectorComponentTest, HdfsDirectoryTest, DistribJoinFromCollectionTest, DeleteNodeTest, CopyFieldTest, TestMultiWordSynonyms, TestSubQueryTransformerCrossCore, TestNRTOpen, TermVectorComponentDistributedTest, TestDistributedStatsComponentCardinality, TemplateUpdateProcessorTest, DistributedVersionInfoTest, FieldMutatingUpdateProcessorTest, AnalysisErrorHandlingTest, SolrIndexConfigTest, TestRestManager, BasicZkTest, PluginInfoTest, TestManagedSchemaThreadSafety, TestValueSourceCache, PreAnalyzedFieldTest, DeleteShardTest, DistributedDebugComponentTest, CreateCollectionCleanupTest, LeaderElectionIntegrationTest, LoggingHandlerTest, TestSchemaManager, VMParamsZkACLAndCredentialsProvidersTest, TestRangeQuery, TestElisionMultitermQuery, TestFieldCacheReopen, InfoHandlerTest, FullHLLTest, DOMUtilTest, MultiThreadedOCPTest, TestOverriddenPrefixQueryForCustomFieldType, TestSchemaResource, TestCorePropertiesReload, OverseerTest, ShardRoutingTest, RecoveryZkTest, TestReplicationHandler, ClusterStateUpdateTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, TestDistributedSearch, AutoCommitTest, TestDistributedGrouping, TestFaceting, TestRecovery, TestJoin, TestReload, HardAutoCommitTest, DistributedTermsComponentTest, TestCoreContainer, SolrCoreTest, StatsComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestSort, TestLazyCores, DirectUpdateHandlerTest, ShowFileRequestHandlerTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, TestTrie, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestOmitPositions, FileBasedSpellCheckerTest, TermsComponentTest, RequestHandlersTest, DocumentBuilderTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestJmxIntegration, TestCollationField, TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, TestLFUCache, HighlighterConfigTest, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestQuerySenderNoQuery, TestStressRecovery, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, MinimalSchemaTest, TestConfig, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, TestFastLRUCache, TestSystemIdResolver, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, TestSolrJ, TestLRUCache, ZkNodePropsTest, FileUtilsTest, TestRTGBase, DistributedIntervalFacetingTest, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AssignTest, CleanupOldIndexTest, CollectionReloadTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DistribDocExpirationUpdateProcessorTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, ReplaceNodeTest, SSLMigrationTest, ShardSplitTest, TestLocalFSCloudBackupRestore, TestLockTree, TestPullReplica, TestSegmentSorting, TestSolrCloudWithSecureImpersonation] [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=5931498DB2FCC92F -Dtests.slow=true -Dtests.locale=uk-UA -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=US-ASCII [junit4] ERROR 0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<< [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: [junit4] > 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) [junit4] > 2) Thread[id=33572, name=jetty-launcher-5459-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) [junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) [junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) [junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) [junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) [junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) [junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) [junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) [junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) [junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) [junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) [junit4] > 3) Thread[id=33648, name=jetty-launcher-5459-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) [junit4] > at __randomizedtesting.SeedInfo.seed([5931498DB2FCC92F]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=33647, name=jetty-launcher-5459-thread-2-SendThread(127.0.0.1:61976), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) [junit4] > at __randomizedtesting.SeedInfo.seed([5931498DB2FCC92F]:0) [junit4] Completed [550/729 (2!)] on J1 in 57.86s, 12 tests, 2 errors <<< FAILURES! [...truncated 43930 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org