Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/527/
4 tests failed. FAILED: org.apache.solr.cloud.OverseerRolesTest.testOverseerRole Error Message: Error from server at https://127.0.0.1:34725/solr: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Not Found</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/admin/collections. Reason: <pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down. at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:331) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:530) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:289) at org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:149) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) </pre> <hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.8.v20171121</a><hr/> </body> </html> Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34725/solr: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Not Found</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/admin/collections. Reason: <pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down. at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:331) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:530) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:289) at org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:149) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) </pre> <hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.8.v20171121</a><hr/> </body> </html> at __randomizedtesting.SeedInfo.seed([206EE6161273345E:C1A51B8229C0028F]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211) at org.apache.solr.cloud.OverseerRolesTest.testOverseerRole(OverseerRolesTest.java:123) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.OverseerRolesTest Error Message: 18 threads leaked from SUITE scope at org.apache.solr.cloud.OverseerRolesTest: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13644, name=zkConnectionManagerCallback-2336-thread-1-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 5) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173) 8) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=13540, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=13647, name=OverseerCollectionConfigSetProcessor-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=13585, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=13648, name=OverseerAutoScalingTriggerThread-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 18 threads leaked from SUITE scope at org.apache.solr.cloud.OverseerRolesTest: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13644, name=zkConnectionManagerCallback-2336-thread-1-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 5) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173) 8) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=13540, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=13647, name=OverseerCollectionConfigSetProcessor-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=13585, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=13648, name=OverseerAutoScalingTriggerThread-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.OverseerRolesTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0) FAILED: org.apache.solr.cloud.cdcr.CdcrVersionReplicationTest.testCdcrDocVersions Error Message: Error from server at http://127.0.0.1:46847/f_u/ld: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000008 Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46847/f_u/ld: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000008 at __randomizedtesting.SeedInfo.seed([206EE6161273345E:D8F8EDB4E015DB42]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211) at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.startServers(BaseCdcrDistributedZkTest.java:594) at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.createSourceCollection(BaseCdcrDistributedZkTest.java:353) at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.baseBefore(BaseCdcrDistributedZkTest.java:179) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 1804 lines...] [junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J2-20180326_065247_4144214870791478663138.sysout [junit4] >>> JVM J2 emitted unexpected output (verbatim) ---- [junit4] codec: Asserting, pf: LuceneFixedGap, dvf: Memory [junit4] <<< JVM J2: EOF ---- [...truncated 10572 lines...] [junit4] Suite: org.apache.solr.cloud.cdcr.CdcrVersionReplicationTest [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/init-core-data-001 [junit4] 2> 371262 WARN (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=210 numCloses=210 [junit4] 2> 371305 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /f_u/ld [junit4] 2> 371344 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 371360 INFO (Thread-279) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 371360 INFO (Thread-279) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 371456 ERROR (Thread-279) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 371515 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkTestServer start zk server on port:43763 [junit4] 2> 371626 INFO (zkConnectionManagerCallback-280-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 371967 INFO (zkConnectionManagerCallback-282-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 371992 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml [junit4] 2> 371994 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml [junit4] 2> 371995 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 372029 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt [junit4] 2> 372030 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt [junit4] 2> 372031 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml [junit4] 2> 372050 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml [junit4] 2> 372051 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json [junit4] 2> 372052 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 372085 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt [junit4] 2> 372087 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt [junit4] 2> 372089 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCdcrDocVersions [junit4] 2> 373442 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 660000ms [junit4] 2> 373448 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ceab7b1{/f_u/ld,null,AVAILABLE} [junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1742164{HTTP/1.1,[http/1.1]}{127.0.0.1:46847} [junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @373568ms [junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/f_u/ld, hostPort=46847, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/cores} [junit4] 2> 373450 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory [junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0 [junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:47.216Z [junit4] 2> 373564 INFO (zkConnectionManagerCallback-284-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 373585 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 373585 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/solr.xml [junit4] 2> 373655 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored [junit4] 2> 373655 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored [junit4] 2> 373656 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter. [junit4] 2> 373692 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr [junit4] 2> 373727 INFO (zkConnectionManagerCallback-288-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 373757 INFO (zkConnectionManagerCallback-290-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 374174 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 374175 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46847_f_u%2Fld [junit4] 2> 374177 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=72345298786844676-127.0.0.1:46847_f_u%2Fld-n_0000000000) starting [junit4] 2> 374232 INFO (zkConnectionManagerCallback-295-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 374234 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready [junit4] 2> 374243 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46847_f_u%2Fld [junit4] 2> 374454 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 374454 INFO (OverseerStateUpdate-72345298786844676-127.0.0.1:46847_f_u%2Fld-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 374823 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 374873 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 374873 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 374874 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/cores [junit4] 2> 378862 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 378918 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 378918 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 378919 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 600000ms [junit4] 2> 378919 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4ddf1fa4{/f_u/ld,null,AVAILABLE} [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5e4ea662{HTTP/1.1,[http/1.1]}{127.0.0.1:34923} [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @379038ms [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, hostContext=/f_u/ld, hostPort=34923, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/cores} [junit4] 2> 378920 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0 [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:52.665Z [junit4] 2> 379121 INFO (zkConnectionManagerCallback-297-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 379256 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 379256 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/solr.xml [junit4] 2> 379329 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored [junit4] 2> 379329 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored [junit4] 2> 379350 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter. [junit4] 2> 379402 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr [junit4] 2> 379439 INFO (zkConnectionManagerCallback-301-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 379516 INFO (zkConnectionManagerCallback-303-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 379552 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 379586 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 379588 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34923_f_u%2Fld [junit4] 2> 379606 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 379621 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 379857 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 380631 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 380683 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 380683 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 380685 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/cores [junit4] 2> 380711 INFO (zkConnectionManagerCallback-308-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 380712 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 380713 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready [junit4] 2> 382172 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 600000ms [junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a3d776{/f_u/ld,null,AVAILABLE} [junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c51bdd2{HTTP/1.1,[http/1.1]}{127.0.0.1:37717} [junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @382309ms [junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard3, hostContext=/f_u/ld, hostPort=37717, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/cores} [junit4] 2> 382191 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory [junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0 [junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:55.937Z [junit4] 2> 382239 INFO (zkConnectionManagerCallback-310-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 382241 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 382241 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/solr.xml [junit4] 2> 382245 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored [junit4] 2> 382245 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored [junit4] 2> 382246 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter. [junit4] 2> 382277 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr [junit4] 2> 382324 INFO (zkConnectionManagerCallback-314-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 382391 INFO (zkConnectionManagerCallback-316-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 382440 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 382441 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 382456 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37717_f_u%2Fld [junit4] 2> 382457 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 382457 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 382475 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 382589 INFO (zkCallback-315-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 382609 INFO (zkCallback-307-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3) [junit4] 2> 383858 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 383896 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 383896 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 383914 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/cores [junit4] 2> 383938 INFO (zkConnectionManagerCallback-321-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 383951 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3) [junit4] 2> 383951 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready [junit4] 2> 385724 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8 [junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0 [junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults [junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 660000ms [junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1800e3a5{/f_u/ld,null,AVAILABLE} [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@26d74f3c{HTTP/1.1,[http/1.1]}{127.0.0.1:43383} [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @385954ms [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard4, hostContext=/f_u/ld, hostPort=43383, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/cores} [junit4] 2> 385836 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0 [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:59.581Z [junit4] 2> 385887 INFO (zkConnectionManagerCallback-323-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 385921 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 385921 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/solr.xml [junit4] 2> 385958 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored [junit4] 2> 385958 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored [junit4] 2> 385959 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter. [junit4] 2> 386031 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr [junit4] 2> 386092 INFO (zkConnectionManagerCallback-327-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 386180 INFO (zkConnectionManagerCallback-329-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 386231 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3) [junit4] 2> 386270 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 386272 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43383_f_u%2Fld [junit4] 2> 386286 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386302 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386302 INFO (zkCallback-307-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386302 INFO (zkCallback-315-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386302 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386395 INFO (zkCallback-320-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 386411 INFO (zkCallback-328-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4) [junit4] 2> 388487 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 388545 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 388545 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed [junit4] 2> 388559 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/cores [junit4] 2> 388677 INFO (zkConnectionManagerCallback-334-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 388686 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4) [junit4] 2> 388687 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready [junit4] 2> 388920 INFO (zkConnectionManagerCallback-338-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 388935 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4) [junit4] 2> 388936 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready [junit4] 2> 388966 INFO (qtp1084406563-1210) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collecti [...truncated too long message...] ad.run(Thread.java:748) [junit4] > 15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) [junit4] > 18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) [junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) [junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) [junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) [junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: [junit4] > 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at java.lang.Object.wait(Native Method) [junit4] > at java.lang.Object.wait(Object.java:502) [junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) [junit4] > 2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) [junit4] > 5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > 8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) [junit4] > 9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0) [junit4] Completed [328/785 (2!)] on J1 in 121.62s, 1 test, 3 errors <<< FAILURES! [...truncated 51066 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
