Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/231/

7 tests failed.
FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithSourceCluster

Error Message:
Captured an uncaught exception in thread: Thread[id=3440, name=Thread-1539, 
state=RUNNABLE, group=TGRP-CdcrBootstrapTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught 
exception in thread: Thread[id=3440, name=Thread-1539, state=RUNNABLE, 
group=TGRP-CdcrBootstrapTest]
        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740:9C82A20ADBE5940A]:0)
Caused by: java.lang.AssertionError: 1
        at __randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)
        at 
org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:191)
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1360)
        at 
org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:705)
        at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:945)
        at 
org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:765)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest

Error Message:
ObjectTracker found 3 object(s) that were not released!!! 
[MockDirectoryWrapper, SolrCore, MockDirectoryWrapper] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)  at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:672)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:779)  at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:88)
  at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:377)
  at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
  at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
  at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:152)
  at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)  
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:318)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:269)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) 
 at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
 at org.eclipse.jetty.server.Server.handle(Server.java:534)  at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)  at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
 at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
  at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
  at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
 at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)  at 
org.apache.solr.core.SolrCore.reload(SolrCore.java:589)  at 
org.apache.solr.core.CoreContainer.reload(CoreContainer.java:944)  at 
org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:765) 
 at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397) 
 at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:757)
  at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:712)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not 
released!!! [MockDirectoryWrapper, SolrCore, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:672)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:779)
        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:88)
        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:377)
        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:152)
        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:318)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:269)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
        at org.apache.solr.core.SolrCore.reload(SolrCore.java:589)
        at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:944)
        at 
org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:765)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:369)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:251)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
        at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:757)
        at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:712)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
        at sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [SolrZkClient] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:184)  
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)  at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)  at 
org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:226)  at 
org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:121)
  at 
org.apache.solr.client.solrj.impl.CloudSolrClient.connect(CloudSolrClient.java:610)
  at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1048)
  at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1037)
  at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)  at 
org.apache.solr.handler.CdcrReplicatorManager.getCheckpoint(CdcrReplicatorManager.java:196)
  at 
org.apache.solr.handler.CdcrReplicatorManager.initLogReaders(CdcrReplicatorManager.java:159)
  at 
org.apache.solr.handler.CdcrReplicatorManager.stateUpdate(CdcrReplicatorManager.java:134)
  at 
org.apache.solr.handler.CdcrStateManager.callback(CdcrStateManager.java:36)  at 
org.apache.solr.handler.CdcrLeaderStateManager.setAmILeader(CdcrLeaderStateManager.java:108)
  at 
org.apache.solr.handler.CdcrLeaderStateManager.checkIfIAmLeader(CdcrLeaderStateManager.java:95)
  at 
org.apache.solr.handler.CdcrLeaderStateManager.access$400(CdcrLeaderStateManager.java:40)
  at 
org.apache.solr.handler.CdcrLeaderStateManager$LeaderStateWatcher.process(CdcrLeaderStateManager.java:150)
  at 
org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:268)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  
at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [SolrZkClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:184)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
        at 
org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:226)
        at 
org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:121)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.connect(CloudSolrClient.java:610)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1048)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1037)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
        at 
org.apache.solr.handler.CdcrReplicatorManager.getCheckpoint(CdcrReplicatorManager.java:196)
        at 
org.apache.solr.handler.CdcrReplicatorManager.initLogReaders(CdcrReplicatorManager.java:159)
        at 
org.apache.solr.handler.CdcrReplicatorManager.stateUpdate(CdcrReplicatorManager.java:134)
        at 
org.apache.solr.handler.CdcrStateManager.callback(CdcrStateManager.java:36)
        at 
org.apache.solr.handler.CdcrLeaderStateManager.setAmILeader(CdcrLeaderStateManager.java:108)
        at 
org.apache.solr.handler.CdcrLeaderStateManager.checkIfIAmLeader(CdcrLeaderStateManager.java:95)
        at 
org.apache.solr.handler.CdcrLeaderStateManager.access$400(CdcrLeaderStateManager.java:40)
        at 
org.apache.solr.handler.CdcrLeaderStateManager$LeaderStateWatcher.process(CdcrLeaderStateManager.java:150)
        at 
org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:268)
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
        at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
2 threads leaked from SUITE scope at 
org.apache.solr.cloud.CdcrReplicationDistributedZkTest:     1) Thread[id=13299, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-EventThread,
 state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    
2) Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.CdcrReplicationDistributedZkTest: 
   1) Thread[id=13299, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-EventThread,
 state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
        at __randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
        at __randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Expected to see collection awhollynewcollection_0 null Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={
   "replicationFactor":"4",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"2",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected to see collection awhollynewcollection_0
null
Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740:DA1877ADEB2A8D5]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:237)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:496)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
KeeperErrorCode = NoNode for /collections/withconfigset2

Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode 
for /collections/withconfigset2
        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740:68AABC942FA53D4C]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testSpecificConfigsets(CollectionsAPIDistributedZkTest.java:425)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11339 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/init-core-data-001
   [junit4]   2> 23197 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 23201 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/tempDir-001
   [junit4]   2> 23218 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 23227 INFO  (Thread-50) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 23228 INFO  (Thread-50) [    ] o.a.s.c.ZkTestServer Starting 
server
   [junit4]   2> 23327 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.c.ZkTestServer start zk server on port:38251
   [junit4]   2> 23482 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to 
java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 25253 INFO  (jetty-launcher-11-thread-3) [    ] o.e.j.s.Server 
jetty-9.3.14.v20161028
   [junit4]   2> 25254 INFO  (jetty-launcher-11-thread-1) [    ] o.e.j.s.Server 
jetty-9.3.14.v20161028
   [junit4]   2> 25262 INFO  (jetty-launcher-11-thread-4) [    ] o.e.j.s.Server 
jetty-9.3.14.v20161028
   [junit4]   2> 25254 INFO  (jetty-launcher-11-thread-2) [    ] o.e.j.s.Server 
jetty-9.3.14.v20161028
   [junit4]   2> 25329 INFO  (jetty-launcher-11-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@271c81fc{/solr,null,AVAILABLE}
   [junit4]   2> 25334 INFO  (jetty-launcher-11-thread-3) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@70838164{HTTP/1.1,[http/1.1]}{127.0.0.1:55784}
   [junit4]   2> 25334 INFO  (jetty-launcher-11-thread-3) [    ] o.e.j.s.Server 
Started @29725ms
   [junit4]   2> 25334 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=55784}
   [junit4]   2> 25335 ERROR (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 25335 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 25335 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 25335 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 25335 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-18T16:54:54.219Z
   [junit4]   2> 25465 INFO  (jetty-launcher-11-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@77a0a2cd{/solr,null,AVAILABLE}
   [junit4]   2> 25475 INFO  (jetty-launcher-11-thread-4) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1b66d217{/solr,null,AVAILABLE}
   [junit4]   2> 25482 INFO  (jetty-launcher-11-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3a4dda0b{/solr,null,AVAILABLE}
   [junit4]   2> 25498 INFO  (jetty-launcher-11-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@6570b9b{HTTP/1.1,[http/1.1]}{127.0.0.1:47833}
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-1) [    ] o.e.j.s.Server 
Started @29892ms
   [junit4]   2> 25498 INFO  (jetty-launcher-11-thread-4) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@6a75f5ac{HTTP/1.1,[http/1.1]}{127.0.0.1:58398}
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-4) [    ] o.e.j.s.Server 
Started @29892ms
   [junit4]   2> 25499 INFO  (jetty-launcher-11-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@2387bc8e{HTTP/1.1,[http/1.1]}{127.0.0.1:33161}
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-2) [    ] o.e.j.s.Server 
Started @29892ms
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=47833}
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=58398}
   [junit4]   2> 25501 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=33161}
   [junit4]   2> 25504 ERROR (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 25505 ERROR (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 25505 ERROR (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 25506 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 25507 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 25507 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 25507 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-18T16:54:54.391Z
   [junit4]   2> 25506 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 25508 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 25508 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 25508 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-18T16:54:54.392Z
   [junit4]   2> 25506 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 25508 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 25509 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 25509 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-18T16:54:54.393Z
   [junit4]   2> 25556 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 25579 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15912db80410001, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 25677 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 25694 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 25701 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 25761 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 25762 INFO  (jetty-launcher-11-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38251/solr
   [junit4]   2> 25777 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 25781 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 25793 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 25821 INFO  (jetty-launcher-11-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38251/solr
   [junit4]   2> 25861 INFO  (jetty-launcher-11-thread-4) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38251/solr
   [junit4]   2> 25909 INFO  (jetty-launcher-11-thread-3) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38251/solr
   [junit4]   2> 26445 INFO  (jetty-launcher-11-thread-2) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:33161_solr
   [junit4]   2> 26451 INFO  (jetty-launcher-11-thread-1) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:47833_solr
   [junit4]   2> 26462 INFO  (jetty-launcher-11-thread-3) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:55784_solr
   [junit4]   2> 26462 INFO  (jetty-launcher-11-thread-2) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.Overseer Overseer 
(id=97129600924057609-127.0.0.1:33161_solr-n_0000000000) starting
   [junit4]   2> 26494 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 26494 INFO  
(zkCallback-29-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 26495 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 26499 INFO  
(zkCallback-29-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 26499 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 26499 INFO  
(zkCallback-31-thread-1-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 26500 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 26599 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 26609 INFO  (jetty-launcher-11-thread-4) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:58398_solr
   [junit4]   2> 26613 INFO  
(zkCallback-29-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 26613 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 26613 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 26613 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 26998 INFO  (jetty-launcher-11-thread-2) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:33161_solr
   [junit4]   2> 27003 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 27004 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 27004 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 27008 INFO  
(zkCallback-29-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 27266 INFO  (jetty-launcher-11-thread-1) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/tempDir-001/node1/.
   [junit4]   2> 27312 INFO  (jetty-launcher-11-thread-4) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/tempDir-001/node4/.
   [junit4]   2> 27324 INFO  (jetty-launcher-11-thread-3) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/tempDir-001/node3/.
   [junit4]   2> 27328 INFO  (jetty-launcher-11-thread-2) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_45D4F3CED8818740-001/tempDir-001/node2/.
   [junit4]   2> 27759 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 27761 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38251/solr ready
   [junit4]   2> 29372 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 31280 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 31740 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 31818 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 31900 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 31992 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_46684_hdfs____hlxk7u/webapp
   [junit4]   2> 33239 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46684
   [junit4]   2> 34319 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 34325 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 34339 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_50271_datanode____m07fqw/webapp
   [junit4]   2> 34846 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:50271
   [junit4]   2> 35865 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 35867 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 35897 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_48080_datanode____krlwjv/webapp
   [junit4]   2> 36637 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[45D4F3CED8818740]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48080
   [junit4]   2> 39623 INFO  (IPC Server handler 8 on 56880) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-580d30a0-75a3-4473-b355-2fd923a32259 node 
DatanodeRegistration(127.0.0.1:56590, 
datanodeUuid=d4ba5c83-c67c-4fa8-bd15-49b07d24b129, infoPort=51310, 
infoSecurePort=0, ipcPort=42938, 
storageInfo=lv=-56;cid=testClusterID;nsid=795121145;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 5 msecs
   [junit4]   2> 39624 INFO  (IPC Server handler 8 on 56880) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-60b8353a-3c36-45cb-8a75-5863f703da92 node 
DatanodeRegistration(127.0.0.1:56590, 
datanodeUuid=d4ba5c83-c67c-4fa8-bd15-49b07d24b129, infoPort=51310, 
infoSecurePort=0, ipcPort=42938, 
storageInfo=lv=-56;cid=testClusterID;nsid=795121145;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 39625 INFO  (IPC Server handler 6 on 56880) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-2775f86c-8bf9-418d-82aa-ab144e012059 node 
DatanodeRegistration(127.0.0.1:52097, 
datanodeUuid=d63b4e3d-24a9-47a0-a9ce-e54987e5fb7d, infoPort=49203, 
infoSecurePort=0, ipcPort=34154, 
storageInfo=lv=-56;cid=testClusterID;nsid=795121145;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 39626 INFO  (IPC Server handler 6 on 56880) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-b3a07fc5-727b-4d95-ac1f-a8ec59759fdc node 
DatanodeRegistration(127.0.0.1:52097, 
datanodeUuid=d63b4e3d-24a9-47a0-a9ce-e54987e5fb7d, infoPort=49203, 
infoSecurePort=0, ipcPort=34154, 
storageInfo=lv=-56;cid=testClusterID;nsid=795121145;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 40120 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[45D4F3CED8818740])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testMaxNodesPerShard
   [junit4]   2> 40122 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[45D4F3CED8818740])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 40170 INFO  (qtp1957733841-139) [n:127.0.0.1:33161_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 40199 INFO  
(OverseerThreadFactory-84-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
oversharded
   [junit4]   2> 40204 ERROR 
(OverseerThreadFactory-84-thread-1-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: oversharded operation: create 
failed:org.apache.solr.common.SolrException: Cannot create collection 
oversharded. Value of maxShardsPerNode is 1, and the number of nodes currently 
live or live and part of your createNodeSet is 4. This allows a maximum of 4 to 
be created. Value of numShards is 3 and value of replicationFactor is 2. This 
requires 6 shards to be created (higher than the allowed number)
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 40264 INFO  (qtp1957733841-139) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2}
 status=400 QTime=122
   [junit4]   2> 40267 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[45D4F3CED8818740])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testMaxNodesPerShard
   [junit4]   2> 40330 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI-seed#[45D4F3CED8818740])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testCollectionsAPI
   [junit4]   2> 40338 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI-seed#[45D4F3CED8818740])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 40344 INFO  (qtp1957733841-140) [n:127.0.0.1:33161_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c&replicationFactor=4&collection.configName=conf2&maxShardsPerNode=2&name=awhollynewcollection_0&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 40347 INFO  (qtp1957733841-140) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c&replicationFactor=4&collection.configName=conf2&maxShardsPerNode=2&name=awhollynewcollection_0&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=3
   [junit4]   2> 40351 INFO  (qtp1957733841-141) [n:127.0.0.1:33161_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
async=7f796b8f-f956-4d36-a92d-f8fd68224824&replicationFactor=3&collection.configName=conf2&maxShardsPerNode=4&name=awhollynewcollection_1&action=CREATE&numShards=4&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 40355 INFO  (qtp1957733841-141) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={async=7f796b8f-f956-4d36-a92d-f8fd68224824&replicationFactor=3&collection.configName=conf2&maxShardsPerNode=4&name=awhollynewcollection_1&action=CREATE&numShards=4&wt=javabin&version=2}
 status=0 QTime=4
   [junit4]   2> 40374 INFO  
(OverseerCollectionConfigSetProcessor-97129600924057609-127.0.0.1:33161_solr-n_0000000000)
 [n:127.0.0.1:33161_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 40378 INFO  
(OverseerThreadFactory-84-thread-2-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
awhollynewcollection_0
   [junit4]   2> 40401 INFO  
(OverseerThreadFactory-84-thread-3-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
awhollynewcollection_1
   [junit4]   2> 40446 INFO  (zkCallback-40-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeCreated 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40973 INFO  (qtp960718685-129) [n:127.0.0.1:55784_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica4&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=37
   [junit4]   2> 40978 INFO  
(parallelCoreAdminExecutor-89-thread-1-processing-n:127.0.0.1:55784_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520 CREATE) 
[n:127.0.0.1:55784_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica4&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 40982 INFO  (qtp1638429046-149) [n:127.0.0.1:58398_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica3&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=48
   [junit4]   2> 40993 INFO  
(parallelCoreAdminExecutor-87-thread-1-processing-n:127.0.0.1:58398_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277 CREATE) 
[n:127.0.0.1:58398_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica3&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41213 INFO  (qtp944827007-157) [n:127.0.0.1:47833_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359452178139&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2}
 status=0 QTime=22
   [junit4]   2> 41219 INFO  
(parallelCoreAdminExecutor-85-thread-1-processing-n:127.0.0.1:47833_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359452178139 CREATE) 
[n:127.0.0.1:47833_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359452178139&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2
   [junit4]   2> 41221 INFO  (qtp1957733841-137) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica2&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=269
   [junit4]   2> 41221 INFO  
(parallelCoreAdminExecutor-91-thread-1-processing-n:127.0.0.1:33161_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608 CREATE) 
[n:127.0.0.1:33161_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica2&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41237 INFO  (qtp944827007-157) [n:127.0.0.1:47833_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359484201352&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2}
 status=0 QTime=22
   [junit4]   2> 41237 INFO  
(parallelCoreAdminExecutor-85-thread-2-processing-n:127.0.0.1:47833_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359484201352 CREATE) 
[n:127.0.0.1:47833_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359484201352&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2
   [junit4]   2> 41306 INFO  (zkCallback-40-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41307 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41307 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41307 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41323 INFO  (qtp1638429046-150) [n:127.0.0.1:58398_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359384672860&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2}
 status=0 QTime=99
   [junit4]   2> 41361 INFO  (qtp960718685-130) [n:127.0.0.1:55784_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359442398387&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2}
 status=0 QTime=166
   [junit4]   2> 41362 INFO  
(parallelCoreAdminExecutor-89-thread-2-processing-n:127.0.0.1:55784_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359442398387 CREATE) 
[n:127.0.0.1:55784_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359442398387&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2
   [junit4]   2> 41373 INFO  (qtp1638429046-147) [n:127.0.0.1:58398_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359463301937&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2}
 status=0 QTime=185
   [junit4]   2> 41405 INFO  (qtp1957733841-136) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359496675840&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2}
 status=0 QTime=96
   [junit4]   2> 41409 INFO  
(parallelCoreAdminExecutor-87-thread-2-processing-n:127.0.0.1:58398_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359384672860 CREATE) 
[n:127.0.0.1:58398_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359384672860&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41409 INFO  
(parallelCoreAdminExecutor-91-thread-2-processing-n:127.0.0.1:33161_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359496675840 CREATE) 
[n:127.0.0.1:33161_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359496675840&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2
   [junit4]   2> 41410 INFO  
(parallelCoreAdminExecutor-87-thread-4-processing-n:127.0.0.1:58398_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359504152651 CREATE) 
[n:127.0.0.1:58398_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359504152651&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2
   [junit4]   2> 41413 INFO  
(parallelCoreAdminExecutor-87-thread-3-processing-n:127.0.0.1:58398_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359463301937 CREATE) 
[n:127.0.0.1:58398_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359463301937&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2
   [junit4]   2> 41417 INFO  (qtp1638429046-151) [n:127.0.0.1:58398_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359504152651&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard2_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard2&wt=javabin&version=2}
 status=0 QTime=10
   [junit4]   2> 41433 INFO  (qtp1957733841-139) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359572617555&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2}
 status=0 QTime=109
   [junit4]   2> 41434 INFO  
(parallelCoreAdminExecutor-85-thread-3-processing-n:127.0.0.1:47833_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359488816146 CREATE) 
[n:127.0.0.1:47833_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359488816146&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2
   [junit4]   2> 41437 INFO  (qtp944827007-160) [n:127.0.0.1:47833_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359488816146&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2}
 status=0 QTime=211
   [junit4]   2> 41457 INFO  
(parallelCoreAdminExecutor-89-thread-4-processing-n:127.0.0.1:55784_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359442573119 CREATE) 
[n:127.0.0.1:55784_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359442573119&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2
   [junit4]   2> 41501 INFO  
(parallelCoreAdminExecutor-91-thread-3-processing-n:127.0.0.1:33161_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359572617555 CREATE) 
[n:127.0.0.1:33161_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359572617555&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard3_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard3&wt=javabin&version=2
   [junit4]   2> 41505 INFO  
(parallelCoreAdminExecutor-91-thread-4-processing-n:127.0.0.1:33161_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359506823806 CREATE) 
[n:127.0.0.1:33161_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359506823806&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41507 INFO  (qtp1957733841-140) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359506823806&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2}
 status=0 QTime=176
   [junit4]   2> 41507 INFO  
(parallelCoreAdminExecutor-85-thread-4-processing-n:127.0.0.1:47833_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358966233972 CREATE) 
[n:127.0.0.1:47833_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358966233972&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica1&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41508 INFO  (qtp944827007-159) [n:127.0.0.1:47833_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358966233972&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica1&action=CREATE&numShards=1&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=134
   [junit4]   2> 41509 INFO  
(parallelCoreAdminExecutor-89-thread-3-processing-n:127.0.0.1:55784_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359468402965 CREATE) 
[n:127.0.0.1:55784_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359468402965&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 41510 INFO  (qtp960718685-126) [n:127.0.0.1:55784_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359468402965&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard1_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard1&wt=javabin&version=2}
 status=0 QTime=96
   [junit4]   2> 41511 INFO  (qtp960718685-131) [n:127.0.0.1:55784_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=7f796b8f-f956-4d36-a92d-f8fd6822482420456359442573119&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_1_shard4_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_1&shard=shard4&wt=javabin&version=2}
 status=0 QTime=96
   [junit4]   2> 41520 INFO  (qtp1957733841-136) [n:127.0.0.1:33161_solr    ] 
o.a.s.h.a.CoreAdminOperation Checking request status for : 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608
   [junit4]   2> 41520 INFO  (qtp1957733841-136) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&requestid=48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608&action=REQUESTSTATUS&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 41521 INFO  (qtp1957733841-136) [n:127.0.0.1:33161_solr    ] 
o.a.s.h.a.CoreAdminOperation Checking request status for : 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359572617555
   [junit4]   2> 41521 INFO  (qtp1957733841-136) [n:127.0.0.1:33161_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&requestid=7f796b8f-f956-4d36-a92d-f8fd6822482420456359572617555&action=REQUESTSTATUS&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 41566 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41567 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41567 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41568 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41582 INFO  (zkCallback-40-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41582 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41583 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41584 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41692 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41730 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41732 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41733 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41733 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41787 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41787 INFO  (zkCallback-40-thread-3) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41787 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41787 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41789 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41919 INFO  
(zkCallback-32-thread-1-processing-n:127.0.0.1:55784_solr) 
[n:127.0.0.1:55784_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41920 INFO  
(zkCallback-31-thread-2-processing-n:127.0.0.1:58398_solr) 
[n:127.0.0.1:58398_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41920 INFO  
(zkCallback-30-thread-1-processing-n:127.0.0.1:47833_solr) 
[n:127.0.0.1:47833_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41923 INFO  
(zkCallback-29-thread-5-processing-n:127.0.0.1:33161_solr) 
[n:127.0.0.1:33161_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/awhollynewcollection_1/state.json] for collection 
[awhollynewcollection_1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 42254 INFO  
(parallelCoreAdminExecutor-89-thread-1-processing-n:127.0.0.1:55784_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520 CREATE) 
[n:127.0.0.1:55784_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica4] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 6.4.0
   [junit4]   2> 42282 INFO  
(parallelCoreAdminExecutor-87-thread-1-processing-n:127.0.0.1:58398_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277 CREATE) 
[n:127.0.0.1:58398_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica3] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 6.4.0
   [junit4]   2> 42348 INFO  
(parallelCoreAdminExecutor-89-thread-1-processing-n:127.0.0.1:55784_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520 CREATE) 
[n:127.0.0.1:55784_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica4] o.a.s.s.IndexSchema 
[awhollynewcollection_0_shard1_replica4] Schema name=minimal
   [junit4]   2> 42377 INFO  
(parallelCoreAdminExecutor-87-thread-1-processing-n:127.0.0.1:58398_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277 CREATE) 
[n:127.0.0.1:58398_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica3] o.a.s.s.IndexSchema 
[awhollynewcollection_0_shard1_replica3] Schema name=minimal
   [junit4]   2> 42380 INFO  
(parallelCoreAdminExecutor-91-thread-1-processing-n:127.0.0.1:33161_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608 CREATE) 
[n:127.0.0.1:33161_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica2] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 6.4.0
   [junit4]   2> 42399 INFO  
(parallelCoreAdminExecutor-89-thread-1-processing-n:127.0.0.1:55784_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520 CREATE) 
[n:127.0.0.1:55784_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica4] o.a.s.s.IndexSchema Loaded schema 
minimal/1.1 with uniqueid field id
   [junit4]   2> 42399 INFO  
(parallelCoreAdminExecutor-85-thread-2-processing-n:127.0.0.1:47833_solr 
7f796b8f-f956-4d36-a92d-f8fd6822482420456359484201352 CREATE) 
[n:127.0.0.1:47833_solr c:awhollynewcollection_1 s:shard2  
x:awhollynewcollection_1_shard2_replica1] o.a.s.c.SolrConfig Using Lucene 
MatchVersion: 6.4.0
   [junit4]   2> 42401 INFO  
(parallelCoreAdminExecutor-89-thread-1-processing-n:127.0.0.1:55784_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358956171520 CREATE) 
[n:127.0.0.1:55784_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica4] o.a.s.c.CoreContainer Creating 
SolrCore 'awhollynewcollection_0_shard1_replica4' using configuration from 
collection awhollynewcollection_0
   [junit4]   2> 42402 INFO  
(parallelCoreAdminExecutor-87-thread-1-processing-n:127.0.0.1:58398_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277 CREATE) 
[n:127.0.0.1:58398_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica3] o.a.s.s.IndexSchema Loaded schema 
minimal/1.1 with uniqueid field id
   [junit4]   2> 42409 INFO  
(parallelCoreAdminExecutor-87-thread-1-processing-n:127.0.0.1:58398_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456358941029277 CREATE) 
[n:127.0.0.1:58398_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica3] o.a.s.c.CoreContainer Creating 
SolrCore 'awhollynewcollection_0_shard1_replica3' using configuration from 
collection awhollynewcollection_0
   [junit4]   2> 42424 INFO  
(parallelCoreAdminExecutor-91-thread-1-processing-n:127.0.0.1:33161_solr 
48e75d5b-d028-4bae-a0b4-7d4ada0f198c20456359340567608 CREATE) 
[n:127.0.0.1:33161_solr c:awhollynewcollection_0 s:shard1  
x:awhollynewcollection_0_shard1_replica2] o.a.s.s.IndexSchema 
[awhollynewcollection_0_shard1_replica2] Schema name=minimal
   [junit4]   2> 42440 INFO  (parallelCoreAdminExecutor-91-thread-1-processing-

[...truncated too long message...]

onReplicationTest, TestSolr4Spatial, ZkNodePropsTest, TestJoin, 
TestFieldTypeResource, TestRandomDVFaceting, SmileWriterTest, 
TestConfigSetsAPI, TestFunctionQuery, DistributedSpellCheckComponentTest, 
ConcurrentDeleteAndCreateCollectionTest, DefaultValueUpdateProcessorTest, 
TestSolrQueryParserDefaultOperatorResource, TestAnalyzeInfixSuggestions, 
TestSolrCloudWithSecureImpersonation, TestFastWriter, 
TriLevelCompositeIdRoutingTest, HttpPartitionTest, VersionInfoTest, 
TestSearcherReuse, TestSubQueryTransformer, TestFoldingMultitermQuery, 
RuleEngineTest, ActionThrottleTest, TestIBSimilarityFactory, 
TestEmbeddedSolrServerConstructors, CurrencyFieldOpenExchangeTest, 
TestDownShardTolerantSearch, CdcrUpdateLogTest, TestCryptoKeys, 
SynonymTokenizerTest, ShardSplitTest, HighlighterMaxOffsetTest, 
WrapperMergePolicyFactoryTest, DocValuesMissingTest, ZkStateReaderTest, 
TestNRTOpen, TestSolrDeletionPolicy2, PathHierarchyTokenizerFactoryTest, 
SegmentsInfoRequestHandlerTest, ParsingFieldUpdateProcessorsTest, 
TestRangeQuery, OverseerRolesTest, HdfsWriteToMultipleCollectionsTest, 
CdcrReplicationDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=CdcrReplicationDistributedZkTest -Dtests.seed=45D4F3CED8818740 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=vi -Dtests.timezone=Indian/Reunion -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | CdcrReplicationDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 
object(s) that were not released!!! [SolrZkClient]
   [junit4]    > 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
   [junit4]    >        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:184)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
   [junit4]    >        at 
org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:226)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:121)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.connect(CloudSolrClient.java:610)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1048)
   [junit4]    >        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1037)
   [junit4]    >        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    >        at 
org.apache.solr.handler.CdcrReplicatorManager.getCheckpoint(CdcrReplicatorManager.java:196)
   [junit4]    >        at 
org.apache.solr.handler.CdcrReplicatorManager.initLogReaders(CdcrReplicatorManager.java:159)
   [junit4]    >        at 
org.apache.solr.handler.CdcrReplicatorManager.stateUpdate(CdcrReplicatorManager.java:134)
   [junit4]    >        at 
org.apache.solr.handler.CdcrStateManager.callback(CdcrStateManager.java:36)
   [junit4]    >        at 
org.apache.solr.handler.CdcrLeaderStateManager.setAmILeader(CdcrLeaderStateManager.java:108)
   [junit4]    >        at 
org.apache.solr.handler.CdcrLeaderStateManager.checkIfIAmLeader(CdcrLeaderStateManager.java:95)
   [junit4]    >        at 
org.apache.solr.handler.CdcrLeaderStateManager.access$400(CdcrLeaderStateManager.java:40)
   [junit4]    >        at 
org.apache.solr.handler.CdcrLeaderStateManager$LeaderStateWatcher.process(CdcrLeaderStateManager.java:150)
   [junit4]    >        at 
org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:268)
   [junit4]    >        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]    >        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE 
scope at org.apache.solr.cloud.CdcrReplicationDistributedZkTest: 
   [junit4]    >    1) Thread[id=13299, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-EventThread,
 state=WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   [junit4]    >    2) Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)Throwable #3: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=13298, 
name=zkCallback-2136-thread-5-processing-n:127.0.0.1:49190_m_sbe%2Fus-SendThread(127.0.0.1:52678),
 state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([45D4F3CED8818740]:0)
   [junit4] Completed [483/662 (3!)] on J0 in 1868.22s, 11 tests, 1 failure, 2 
errors <<< FAILURES!

[...truncated 44827 lines...]
-documentation-lint:
     [echo] checking for broken html...
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] 
file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.DoubleDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
     [exec] 
     [exec] 
file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.LongDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
     [exec] 
     [exec] 
file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.SortedLongDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
     [exec] 
     [exec] 
file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.SortedDoubleDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
     [exec]   BROKEN LINK: 
file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
     [exec] 
     [exec] Broken javadocs links were found!

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:773:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:101:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build.xml:138:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build.xml:151:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:2478:
 exec returned: 1

Total time: 292 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to