Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/336/

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [RawDirectoryWrapper] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.RawDirectoryWrapper  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:427)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:301) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:400) 
 at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:757)
  at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:712)
  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.RawDirectoryWrapper
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:427)
        at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:301)
        at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:400)
        at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:757)
        at 
org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:712)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([41CAEDCB8C21E182]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
        at sun.reflect.GeneratedMethodAccessor163.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:58687/oneInstanceCollection22: Expected 
mime type application/octet-stream but got text/html. <html> <head> <meta 
http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 
500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing 
/oneInstanceCollection22/update. Reason: <pre>    
{metadata={error-class=org.apache.solr.core.SolrCoreInitializationException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore
 &apos;oneInstanceCollection22&apos; is not available due to init failure: The 
max direct memory is likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 
MB.,trace=org.apache.solr.core.SolrCoreInitializationException: SolrCore 
&apos;oneInstanceCollection22&apos; is not available due to init failure: The 
max direct memory is likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 MB.  at 
org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1309)  at 
org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:268)  at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:483)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:344)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:295)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) 
 at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
 at org.eclipse.jetty.server.Server.handle(Server.java:534)  at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)  at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
 at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
  at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
  at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
 at java.lang.Thread.run(Thread.java:745) Caused by: 
org.apache.solr.common.SolrException: The max direct memory is likely too low.  
Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g 
-XX:+UseLargePages to your containers startup args) or disable direct 
allocation using solr.hdfs.blockcache.direct.memory.allocation=false in 
solrconfig.xml. If you are putting the block cache on the heap, your java heap 
size might not be large enough. Failed allocating ~8.388608 MB.  at 
org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:972)  at 
org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:829)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:916)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:851)  at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
  at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:378)
  at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
  at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
  at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)  
at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)  
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:344)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:295)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) 
 at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)  
... 13 more Caused by: java.lang.RuntimeException: The max direct memory is 
likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 MB.  at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:310)
  at 
org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:286)
  at 
org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:226) 
 at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:91)  at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:727)  at 
org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:918)  ... 37 more 
Caused by: java.lang.OutOfMemoryError: Direct buffer memory  at 
java.nio.Bits.reserveMemory(Bits.java:693)  at 
java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)  at 
java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)  at 
org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:75)  
at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:308)
  ... 43 more ,code=500}</pre></p> <hr /><a 
href="http://eclipse.org/jetty";>Powered by Jetty:// 9.3.14.v20161028</a><hr/> 
</body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:58687/oneInstanceCollection22: Expected mime 
type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /oneInstanceCollection22/update. Reason:
<pre>    
{metadata={error-class=org.apache.solr.core.SolrCoreInitializationException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore
 &apos;oneInstanceCollection22&apos; is not available due to init failure: The 
max direct memory is likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 
MB.,trace=org.apache.solr.core.SolrCoreInitializationException: SolrCore 
&apos;oneInstanceCollection22&apos; is not available due to init failure: The 
max direct memory is likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 MB.
        at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1309)
        at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:268)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:483)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:344)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:295)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is 
likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers 
startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~8.388608 MB.
        at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:972)
        at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:829)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:916)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:851)
        at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
        at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:378)
        at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
        at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
        at 
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:344)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:295)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
        ... 13 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low. 
 Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g 
-XX:+UseLargePages to your containers startup args) or disable direct 
allocation using solr.hdfs.blockcache.direct.memory.allocation=false in 
solrconfig.xml. If you are putting the block cache on the heap, your java heap 
size might not be large enough. Failed allocating ~8.388608 MB.
        at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:310)
        at 
org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:286)
        at 
org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:226)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
        at 
org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:91)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:727)
        at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:918)
        ... 37 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
        at java.nio.Bits.reserveMemory(Bits.java:693)
        at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)
        at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
        at 
org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:75)
        at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:308)
        ... 43 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 
9.3.14.v20161028</a><hr/>
</body>
</html>

        at 
__randomizedtesting.SeedInfo.seed([41CAEDCB8C21E182:C99ED21122DD8C7A]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:578)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
        at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
        at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
        at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
        at 
org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:799)
        at 
org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:361)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)  at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3002)  
at org.apache.solr.core.SolrCore.close(SolrCore.java:1545)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:964)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:916)  at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:555)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3002)
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1545)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:964)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:916)
        at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:555)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([41CAEDCB8C21E182]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
        at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1493558942233,generation=2,filelist=[_nr.cfe, _nr.cfs, 
_nr.si, _ns.cfe, _ns.cfs, _ns.si, _nt.cfe, _nt.cfs, _nt.si, _nu.cfe, _nu.cfs, 
_nu.si, _nv.cfe, _nv.cfs, _nv.si, _ny.fdt, _ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, 
_ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, _nz.si, segments_2]}]> but 
was:<[{indexVersion=1493558942233,generation=2,filelist=[_nr.cfe, _nr.cfs, 
_nr.si, _ns.cfe, _ns.cfs, _ns.si, _nt.cfe, _nt.cfs, _nt.si, _nu.cfe, _nu.cfs, 
_nu.si, _nv.cfe, _nv.cfs, _nv.si, _ny.fdt, _ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, 
_ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, _nz.si, segments_2]}, 
{indexVersion=1493558942233,generation=3,filelist=[_nr.cfe, _nr.cfs, _nr.si, 
_nx.fdt, _nx.fdx, _nx.fnm, _nx.nvd, _nx.nvm, _nx.si, _nx_Memory_0.ram, _ny.fdt, 
_ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, _ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, 
_nz.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: 
expected:<[{indexVersion=1493558942233,generation=2,filelist=[_nr.cfe, _nr.cfs, 
_nr.si, _ns.cfe, _ns.cfs, _ns.si, _nt.cfe, _nt.cfs, _nt.si, _nu.cfe, _nu.cfs, 
_nu.si, _nv.cfe, _nv.cfs, _nv.si, _ny.fdt, _ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, 
_ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, _nz.si, segments_2]}]> but 
was:<[{indexVersion=1493558942233,generation=2,filelist=[_nr.cfe, _nr.cfs, 
_nr.si, _ns.cfe, _ns.cfs, _ns.si, _nt.cfe, _nt.cfs, _nt.si, _nu.cfe, _nu.cfs, 
_nu.si, _nv.cfe, _nv.cfs, _nv.si, _ny.fdt, _ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, 
_ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, _nz.si, segments_2]}, 
{indexVersion=1493558942233,generation=3,filelist=[_nr.cfe, _nr.cfs, _nr.si, 
_nx.fdt, _nx.fdx, _nx.fnm, _nx.nvd, _nx.nvm, _nx.si, _nx_Memory_0.ram, _ny.fdt, 
_ny.fdx, _ny.fnm, _ny.nvd, _ny.nvm, _ny.si, _ny_Memory_0.ram, _nz.cfe, _nz.cfs, 
_nz.si, segments_3]}]>
        at 
__randomizedtesting.SeedInfo.seed([41CAEDCB8C21E182:641DF6FBFC69EF81]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1281)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11914 lines...]
   [junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/init-core-data-001
   [junit4]   2> 563198 WARN  
(SUITE-CdcrBootstrapTest-seed#[41CAEDCB8C21E182]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4220 numCloses=4220
   [junit4]   2> 563198 INFO  
(SUITE-CdcrBootstrapTest-seed#[41CAEDCB8C21E182]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 563200 INFO  
(SUITE-CdcrBootstrapTest-seed#[41CAEDCB8C21E182]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 563203 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithSourceCluster
   [junit4]   2> 563203 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-target-001
   [junit4]   2> 563203 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 563204 INFO  (Thread-10959) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 563204 INFO  (Thread-10959) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 563208 ERROR (Thread-10959) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 563304 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.ZkTestServer start zk server on port:40701
   [junit4]   2> 563315 INFO  (jetty-launcher-299-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 563319 INFO  (jetty-launcher-299-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@74ed4eba{/solr,null,AVAILABLE}
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@26489ae3{HTTP/1.1,[http/1.1]}{127.0.0.1:58887}
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.e.j.s.Server Started @566611ms
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=58887}
   [junit4]   2> 563330 ERROR (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.6.0
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 563330 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-30T12:59:22.175Z
   [junit4]   2> 563339 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 563348 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 563349 INFO  (jetty-launcher-299-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40701/solr
   [junit4]   2> 563379 INFO  (jetty-launcher-299-thread-1) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 563380 INFO  (jetty-launcher-299-thread-1) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:58887_solr
   [junit4]   2> 563381 INFO  (jetty-launcher-299-thread-1) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.Overseer Overseer 
(id=97881762172174339-127.0.0.1:58887_solr-n_0000000000) starting
   [junit4]   2> 563394 INFO  (jetty-launcher-299-thread-1) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:58887_solr
   [junit4]   2> 563395 INFO  
(zkCallback-306-thread-1-processing-n:127.0.0.1:58887_solr) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 563465 INFO  (jetty-launcher-299-thread-1) 
[n:127.0.0.1:58887_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-target-001/node1/.
   [junit4]   1> Target zkHost = 127.0.0.1:40701/solr
   [junit4]   2> 563529 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-source-001
   [junit4]   2> 563529 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 563530 INFO  (Thread-10969) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 563530 INFO  (Thread-10969) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 563534 ERROR (Thread-10969) [    ] o.a.z.s.ZooKeeperServer 
ZKShutdownHandler is not registered, so ZooKeeper server won't take any action 
on ERROR or SHUTDOWN server state changes
   [junit4]   2> 563630 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.ZkTestServer start zk server on port:58199
   [junit4]   2> 563647 INFO  (jetty-launcher-311-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 563651 INFO  (jetty-launcher-311-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1da48a8b{/solr,null,AVAILABLE}
   [junit4]   2> 563651 INFO  (jetty-launcher-311-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@34482dfc{HTTP/1.1,[http/1.1]}{127.0.0.1:51723}
   [junit4]   2> 563651 INFO  (jetty-launcher-311-thread-1) [    ] 
o.e.j.s.Server Started @566932ms
   [junit4]   2> 563651 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=51723}
   [junit4]   2> 563651 ERROR (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 563653 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.6.0
   [junit4]   2> 563653 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 563653 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 563653 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-30T12:59:22.498Z
   [junit4]   2> 563664 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 563672 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 563673 INFO  (jetty-launcher-311-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58199/solr
   [junit4]   2> 563708 INFO  (jetty-launcher-311-thread-1) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 563709 INFO  (jetty-launcher-311-thread-1) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:51723_solr
   [junit4]   2> 563710 INFO  (jetty-launcher-311-thread-1) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.Overseer Overseer 
(id=97881762193539075-127.0.0.1:51723_solr-n_0000000000) starting
   [junit4]   2> 563720 INFO  (jetty-launcher-311-thread-1) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:51723_solr
   [junit4]   2> 563723 INFO  
(zkCallback-318-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 563816 INFO  (jetty-launcher-311-thread-1) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-source-001/node1/.
   [junit4]   2> 563920 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 563921 INFO  
(TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[41CAEDCB8C21E182]) 
[    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58199/solr 
ready
   [junit4]   2> 563925 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 563932 INFO  
(OverseerThreadFactory-1131-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
cdcr-source
   [junit4]   2> 564039 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&name=cdcr-source_shard1_replica1&action=CREATE&numShards=1&collection=cdcr-source&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2
   [junit4]   2> 564040 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr    ] 
o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 
transient cores
   [junit4]   2> 564147 INFO  
(zkCallback-318-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/cdcr-source/state.json] for collection [cdcr-source] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 565063 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.6.0
   [junit4]   2> 565070 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but 
managed schema resource managed-schema not found - loading non-managed schema 
schema.xml instead
   [junit4]   2> 565073 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.s.IndexSchema 
[cdcr-source_shard1_replica1] Schema name=minimal
   [junit4]   2> 565076 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 565077 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at 
/configs/cdcr-source/managed-schema
   [junit4]   2> 565079 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in 
ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to 
/configs/cdcr-source/schema.xml.bak
   [junit4]   2> 565080 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.CoreContainer 
Creating SolrCore 'cdcr-source_shard1_replica1' using configuration from 
collection cdcr-source, trusted=true
   [junit4]   2> 565086 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore 
solr.RecoveryStrategy.Builder
   [junit4]   2> 565086 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore 
[[cdcr-source_shard1_replica1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-source-001/node1/cdcr-source_shard1_replica1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_41CAEDCB8C21E182-001/cdcr-source-001/node1/./cdcr-source_shard1_replica1/data/]
   [junit4]   2> 565181 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 565181 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 565182 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 565182 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 565184 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@38a8105c[cdcr-source_shard1_replica1] main]
   [junit4]   2> 565185 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/cdcr-source
   [junit4]   2> 565186 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/cdcr-source
   [junit4]   2> 565186 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at 
/configs/cdcr-source/managed-schema
   [junit4]   2> 565186 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 565186 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 565194 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.h.CdcrBufferStateManager Created znode 
/collections/cdcr-source/cdcr/state/buffer
   [junit4]   2> 565206 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.h.CdcrProcessStateManager Created znode 
/collections/cdcr-source/cdcr/state/process
   [junit4]   2> 565239 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore 
[cdcr-source_shard1_replica1] Registered new searcher 
Searcher@38a8105c[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 565239 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.u.UpdateLog Could 
not find max version in index or recent updates, using new clock 
1566108196886544384
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:51723/solr/cdcr-source_shard1_replica1/
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy 
Sync Success - now sync replicas to me
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy 
http://127.0.0.1:51723/solr/cdcr-source_shard1_replica1/ has no replicas
   [junit4]   2> 565250 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 565253 INFO  
(zkCallback-318-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.h.CdcrLeaderStateManager Received new leader 
state @ cdcr-source:shard1
   [junit4]   2> 565258 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:51723/solr/cdcr-source_shard1_replica1/ shard1
   [junit4]   2> 565359 INFO  
(zkCallback-318-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/cdcr-source/state.json] for collection [cdcr-source] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 565409 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 565411 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1  x:cdcr-source_shard1_replica1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&name=cdcr-source_shard1_replica1&action=CREATE&numShards=1&collection=cdcr-source&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2}
 status=0 QTime=1372
   [junit4]   2> 565418 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 565519 INFO  
(zkCallback-318-thread-1-processing-n:127.0.0.1:51723_solr) 
[n:127.0.0.1:51723_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/cdcr-source/state.json] for collection [cdcr-source] has 
occurred - updating... (live nodes size: [1])
   [junit4]   2> 565933 INFO  
(OverseerCollectionConfigSetProcessor-97881762193539075-127.0.0.1:51723_solr-n_0000000000)
 [n:127.0.0.1:51723_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 566418 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2}
 status=0 QTime=2493
   [junit4]   1> Adding 100 docs with commit=true, numDocs=100
   [junit4]   2> 566509 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=83
   [junit4]   2> 566512 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566512 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566599 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@62cd7628[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566599 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@62cd7628[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)))}
   [junit4]   2> 566600 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566600 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=88
   [junit4]   1> Adding 100 docs with commit=true, numDocs=200
   [junit4]   2> 566624 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=18
   [junit4]   2> 566634 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566634 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566676 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@51d34025[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566676 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@51d34025[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100)))}
   [junit4]   2> 566677 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566677 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=42
   [junit4]   1> Adding 100 docs with commit=true, numDocs=300
   [junit4]   2> 566694 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=12
   [junit4]   2> 566696 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566696 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566719 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@901c8f[cdcr-source_shard1_replica1] 
main]
   [junit4]   2> 566720 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566720 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@901c8f[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100)))}
   [junit4]   2> 566720 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=23
   [junit4]   1> Adding 100 docs with commit=true, numDocs=400
   [junit4]   2> 566728 INFO  (qtp992913987-13122) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=6
   [junit4]   2> 566730 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566730 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566750 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@6e98b12[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566750 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@6e98b12[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100)))}
   [junit4]   2> 566751 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566751 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=20
   [junit4]   1> Adding 100 docs with commit=true, numDocs=500
   [junit4]   2> 566759 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=6
   [junit4]   2> 566761 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566761 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566787 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@6592ef61[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566788 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@6592ef61[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100)))}
   [junit4]   2> 566788 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566788 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=26
   [junit4]   1> Adding 100 docs with commit=true, numDocs=600
   [junit4]   2> 566795 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 566797 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566797 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566819 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@3dc45525[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566820 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@3dc45525[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100)))}
   [junit4]   2> 566820 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566820 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=22
   [junit4]   1> Adding 100 docs with commit=true, numDocs=700
   [junit4]   2> 566828 INFO  (qtp992913987-13122) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 566830 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566830 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566866 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@be38e81[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566867 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@be38e81[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100)))}
   [junit4]   2> 566867 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566867 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=37
   [junit4]   1> Adding 100 docs with commit=true, numDocs=800
   [junit4]   2> 566875 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 566881 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566881 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566904 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@4fa5f041[cdcr-source_shard1_replica1] main]
   [junit4]   2> 566905 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@4fa5f041[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100)))}
   [junit4]   2> 566905 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566905 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=24
   [junit4]   1> Adding 100 docs with commit=true, numDocs=900
   [junit4]   2> 566914 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 566917 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566917 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 566948 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@37c317[cdcr-source_shard1_replica1] 
main]
   [junit4]   2> 566948 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@37c317[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100) Uninverting(_8(6.6.0):C100)))}
   [junit4]   2> 566948 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 566948 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=31
   [junit4]   1> Adding 100 docs with commit=true, numDocs=1000
   [junit4]   2> 566956 INFO  (qtp992913987-13122) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 566963 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 566963 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 567006 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@2d91dc9e[cdcr-source_shard1_replica1] main]
   [junit4]   2> 567007 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@2d91dc9e[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100) Uninverting(_8(6.6.0):C100) 
Uninverting(_9(6.6.0):C100)))}
   [junit4]   2> 567007 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 567007 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=44
   [junit4]   1> Adding 100 docs with commit=true, numDocs=1100
   [junit4]   2> 567018 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=8
   [junit4]   2> 567020 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 567020 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 567053 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@37b0dc1d[cdcr-source_shard1_replica1] main]
   [junit4]   2> 567054 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@37b0dc1d[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100) Uninverting(_8(6.6.0):C100) 
Uninverting(_9(6.6.0):C100) Uninverting(_a(6.6.0):C100)))}
   [junit4]   2> 567054 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 567054 INFO  (qtp992913987-13116) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=34
   [junit4]   1> Adding 100 docs with commit=true, numDocs=1200
   [junit4]   2> 567062 INFO  (qtp992913987-13117) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 567064 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 567064 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 567118 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@5e0940f9[cdcr-source_shard1_replica1] main]
   [junit4]   2> 567119 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 567119 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@5e0940f9[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100) Uninverting(_8(6.6.0):C100) 
Uninverting(_9(6.6.0):C100) Uninverting(_a(6.6.0):C100) 
Uninverting(_c(6.6.0):C100)))}
   [junit4]   2> 567119 INFO  (qtp992913987-13119) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=55
   [junit4]   1> Adding 100 docs with commit=true, numDocs=1300
   [junit4]   2> 567127 INFO  (qtp992913987-13122) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 567129 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 567129 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 567158 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@10d454b[cdcr-source_shard1_replica1] main]
   [junit4]   2> 567159 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@10d454b[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.6.0):C100)
 Uninverting(_1(6.6.0):C100) Uninverting(_2(6.6.0):C100) 
Uninverting(_3(6.6.0):C100) Uninverting(_4(6.6.0):C100) 
Uninverting(_5(6.6.0):C100) Uninverting(_6(6.6.0):C100) 
Uninverting(_7(6.6.0):C100) Uninverting(_8(6.6.0):C100) 
Uninverting(_9(6.6.0):C100) Uninverting(_a(6.6.0):C100) 
Uninverting(_c(6.6.0):C100) Uninverting(_d(6.6.0):C100)))}
   [junit4]   2> 567166 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 567166 INFO  (qtp992913987-13120) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}
 status=0 QTime=36
   [junit4]   1> Adding 100 docs with commit=true, numDocs=1400
   [junit4]   2> 567173 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.c.S.Request [cdcr-source_shard1_replica1]  webapp=/solr path=/update 
params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 567177 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 567177 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.SolrIndexWriter Calling setCommitData with 
IW:org.apache.solr.update.SolrIndexWriter@25b02280
   [junit4]   2> 567263 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@455af48c[cdcr-source_shard1_replica1] main]
   [junit4]   2> 567263 INFO  
(searcherExecutor-1134-thread-1-processing-n:127.0.0.1:51723_solr 
x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) 
[n:127.0.0.1:51723_solr c:cdcr-source s:shard1 r:core_node1 
x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] 
Registered new searcher Searcher@455af48c[cdcr-source_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_b(6.6.0):C1000)
 Uninverting(_a(6.6.0):C100) Uninverting(_c(6.6.0):C100) 
Uninverting(_d(6.6.0):C100) Uninverting(_e(6.6.0):C100)))}
   [junit4]   2> 567263 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr 
c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 567272 INFO  (qtp992913987-13123) [n:127.0.0.1:51723_solr c

[...truncated too long message...]

m.read(SocketInputStream.java:141)
   [junit4]   2>     
java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
   [junit4]   2>     
java.io.BufferedInputStream.read(BufferedInputStream.java:265)
   [junit4]   2>     java.io.FilterInputStream.read(FilterInputStream.java:83)
   [junit4]   2>     [...sun.*]
   [junit4]   2>     java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>     [...sun.*]
   [junit4]   2>     
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>     
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>     java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> RMI TCP Accept-33388:
   [junit4]   2>     java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2>     
java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
   [junit4]   2>     java.net.ServerSocket.implAccept(ServerSocket.java:545)
   [junit4]   2>     java.net.ServerSocket.accept(ServerSocket.java:513)
   [junit4]   2>     [...sun.*]
   [junit4]   2>     java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 3613048 INFO  
(SUITE-HdfsNNFailoverTest-seed#[41CAEDCB8C21E182]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ------------------------------------------------------- 
Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), 
sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=th, 
timezone=America/Guayaquil
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_121 (64-bit)/cpus=4,threads=2,free=320235968,total=529530880
   [junit4]   2> NOTE: All tests run in this JVM: [TestFunctionQuery, 
TestCollationFieldDocValues, TestPerFieldSimilarityClassic, 
RegexBoostProcessorTest, DocExpirationUpdateProcessorFactoryTest, 
SynonymTokenizerTest, TestQuerySenderNoQuery, RequiredFieldsTest, 
TestSystemCollAutoCreate, TestConfigOverlay, TestCustomSort, TestStressLucene, 
TestExportWriter, TestPseudoReturnFields, DistanceUnitsTest, TestFastLRUCache, 
SubstringBytesRefFilterTest, TestPKIAuthenticationPlugin, 
TestTrackingShardHandlerFactory, SSLMigrationTest, ZkStateReaderTest, 
QueryElevationComponentTest, TestSolrCoreProperties, HdfsDirectoryFactoryTest, 
NodeMutatorTest, TestAtomicUpdateErrorCases, TestMinMaxOnMultiValuedField, 
TestDistributedStatsComponentCardinality, TestUnifiedSolrHighlighter, 
RecoveryAfterSoftCommitTest, TestMaxScoreQueryParser, TestFastWriter, 
DefaultValueUpdateProcessorTest, TestJettySolrRunner, TestSolrDynamicMBean, 
PeerSyncTest, AnalysisAfterCoreReloadTest, TestComponentsName, 
ClassificationUpdateProcessorIntegrationTest, TestBinaryResponseWriter, 
HdfsWriteToMultipleCollectionsTest, TestSolrCoreParser, 
ExternalFileFieldSortTest, CursorPagingTest, TestCrossCoreJoin, 
HdfsCollectionsAPIDistributedZkTest, DistributedQueryComponentOptimizationTest, 
AlternateDirectoryTest, CustomCollectionTest, TestSolrDeletionPolicy2, 
TestRTimerTree, TestClassicSimilarityFactory, TestRealTimeGet, TestInitParams, 
StandardRequestHandlerTest, TestFieldCacheWithThreads, TestFaceting, 
LeaderElectionContextKeyTest, XsltUpdateRequestHandlerTest, 
TestStressUserVersions, TestCloudPivotFacet, 
OverseerCollectionConfigSetProcessorTest, ResourceLoaderTest, 
FastVectorHighlighterTest, DeleteNodeTest, TestExclusionRuleCollectionAccess, 
DistributedFacetPivotSmallTest, DisMaxRequestHandlerTest, TestSchemaManager, 
TestFieldTypeResource, ChaosMonkeyNothingIsSafeTest, ReplicationFactorTest, 
TestJmxIntegration, QueryEqualityTest, SolrJmxReporterTest, TestNumericTerms64, 
TestIBSimilarityFactory, SolrCoreMetricManagerTest, BasicDistributedZkTest, 
TestJsonRequest, NumericFieldsTest, TestDefaultStatsCache, DateFieldTest, 
CdcrReplicationHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, 
TestCodecSupport, TestDefaultSearchFieldResource, 
PeerSyncWithIndexFingerprintCachingTest, HLLSerializationTest, 
TestSolrQueryParserDefaultOperatorResource, TestDynamicLoading, 
CheckHdfsIndexTest, TestFileDictionaryLookup, TestBulkSchemaAPI, 
TestGraphTermsQParserPlugin, TestSolr4Spatial, TestConfigSets, 
TestExactStatsCache, AnalysisErrorHandlingTest, TestNamedUpdateProcessors, 
HdfsRecoveryZkTest, DistribJoinFromCollectionTest, TestRangeQuery, 
TestExceedMaxTermLength, SolrCoreTest, TestLockTree, ReturnFieldsTest, 
LeaderInitiatedRecoveryOnCommitTest, TestRandomFaceting, ZkNodePropsTest, 
RankQueryTest, TestClusterProperties, DistributedDebugComponentTest, 
ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, 
SyncSliceTest, LeaderElectionIntegrationTest, ShardRoutingTest, 
FullSolrCloudDistribCmdsTest, TestRandomDVFaceting, ZkCLITest, 
ShardRoutingCustomTest, TestDistributedSearch, TestDistributedGrouping, 
TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, 
TermVectorComponentDistributedTest, ZkControllerTest, TestJoin, TestReload, 
TestStressVersions, DistributedTermsComponentTest, TestCoreContainer, 
StatsComponentTest, SpellCheckComponentTest, TestGroupingSearch, 
ConvertedLegacyTest, TestSort, BasicFunctionalityTest, TestLazyCores, 
TestBadConfig, HighlighterTest, CurrencyFieldOpenExchangeTest, 
DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, 
SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, SolrRequestParserTest, 
TestFoldingMultitermQuery, TestTrie, TestCSVLoader, PolyFieldTest, 
NoCacheHeaderTest, TestRemoteStreaming, DocValuesMultiTest, 
DistanceFunctionTest, TestSolrDeletionPolicy1, DebugComponentTest, 
CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, 
TestQueryUtils, TestWriterPerf, TestReversedWildcardFilterFactory, 
DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TestOmitPositions, 
FileBasedSpellCheckerTest, TestValueSourceCache, 
PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, 
MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, 
SolrPluginUtilsTest, MBeansHandlerTest, QueryParsingTest, 
TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, 
TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, 
CSVRequestHandlerTest, HighlighterConfigTest, UpdateParamsTest, 
TestSolrIndexConfig, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, 
SliceStateTest, SystemInfoHandlerTest, CircularListTest, 
DistributedMLTComponentTest, SolrTestCaseJ4Test, 
TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, 
TestHighlightDedupGrouping, TestPathTrie, TestEmbeddedSolrServerConstructors, 
TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, ActionThrottleTest, 
AliasIntegrationTest, BaseCdcrDistributedZkTest, 
CdcrReplicationDistributedZkTest, CdcrRequestHandlerTest, 
CollectionsAPISolrJTest, ConfigSetsAPITest, DeleteInactiveReplicaTest, 
DeleteReplicaTest, DeleteStatusTest, DocValuesNotIndexedTest, 
HttpPartitionTest, MultiThreadedOCPTest, SharedFSAutoReplicaFailoverUtilsTest, 
SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, 
TestAuthenticationFramework, TestCloudDeleteByQuery, 
TestCloudPseudoReturnFields, TestCloudRecovery, 
TestDeleteCollectionOnDownNodes, TestDownShardTolerantSearch, 
TestHdfsCloudBackupRestore, TestMiniSolrCloudClusterSSL, TestRequestForwarding, 
TestSSLRandomization, TestShortCircuitedRequests, 
TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, 
HdfsBasicDistributedZkTest, HdfsNNFailoverTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 
'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest 
-Dtests.seed=41CAEDCB8C21E182 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=th -Dtests.timezone=America/Guayaquil -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | HdfsNNFailoverTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 
object(s) that were not released!!! [NRTCachingDirectory]
   [junit4]    > 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory
   [junit4]    >        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    >        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3002)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.close(SolrCore.java:1545)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:964)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:916)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:555)
   [junit4]    >        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    >        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([41CAEDCB8C21E182]:0)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [544/708 (4!)] on J0 in 56.96s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 43899 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to