[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1295 - Failure

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1295/

7 tests failed.
FAILED:  org.apache.solr.TestGroupingSearch.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.util.Arrays.copyOfRange(Arrays.java:3664)
        at java.lang.String.<init>(String.java:207)
        at java.lang.StringBuilder.toString(StringBuilder.java:407)
        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:187)
        at java.lang.reflect.Method.toGenericString(Method.java:415)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
        at java.util.TimSort.binarySort(TimSort.java:296)
        at java.util.TimSort.sort(TimSort.java:239)
        at java.util.Arrays.sort(Arrays.java:1438)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.<init>(RandomizedRunner.java:329)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at 
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at 
org.junit.internal.builders.AnnotatedBuilder.buildRunner(AnnotatedBuilder.java:31)
        at 
org.junit.internal.builders.AnnotatedBuilder.runnerForClass(AnnotatedBuilder.java:24)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.builders.AllDefaultPossibilitiesBuilder.runnerForClass(AllDefaultPossibilitiesBuilder.java:29)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.requests.ClassRequest.getRunner(ClassRequest.java:24)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:239)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)


FAILED:  org.apache.solr.cloud.TestSegmentSorting.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.util.Arrays.copyOf(Arrays.java:3332)
        at 
java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
        at 
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
        at java.lang.StringBuilder.append(StringBuilder.java:136)
        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:171)
        at java.lang.reflect.Method.toGenericString(Method.java:415)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
        at java.util.TimSort.binarySort(TimSort.java:296)
        at java.util.TimSort.sort(TimSort.java:239)
        at java.util.Arrays.sort(Arrays.java:1438)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.<init>(RandomizedRunner.java:329)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at 
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at 
org.junit.internal.builders.AnnotatedBuilder.buildRunner(AnnotatedBuilder.java:31)
        at 
org.junit.internal.builders.AnnotatedBuilder.runnerForClass(AnnotatedBuilder.java:24)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.builders.AllDefaultPossibilitiesBuilder.runnerForClass(AllDefaultPossibilitiesBuilder.java:29)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.requests.ClassRequest.getRunner(ClassRequest.java:24)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:239)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)


FAILED:  
org.apache.solr.handler.admin.ShowFileRequestHandlerTest.initializationError

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.lang.reflect.Modifier.toString(Modifier.java:235)
        at 
java.lang.reflect.Executable.printModifiersIfNonzero(Executable.java:100)
        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:146)
        at java.lang.reflect.Method.toGenericString(Method.java:415)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
        at java.util.TimSort.binarySort(TimSort.java:296)
        at java.util.TimSort.sort(TimSort.java:239)
        at java.util.Arrays.sort(Arrays.java:1438)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.<init>(RandomizedRunner.java:329)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at 
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at 
org.junit.internal.builders.AnnotatedBuilder.buildRunner(AnnotatedBuilder.java:31)
        at 
org.junit.internal.builders.AnnotatedBuilder.runnerForClass(AnnotatedBuilder.java:24)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.builders.AllDefaultPossibilitiesBuilder.runnerForClass(AllDefaultPossibilitiesBuilder.java:29)
        at 
org.junit.runners.model.RunnerBuilder.safeRunnerForClass(RunnerBuilder.java:57)
        at 
org.junit.internal.requests.ClassRequest.getRunner(ClassRequest.java:24)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:239)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
        at 
com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)


FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
Timeout occured while waiting response from server at: 
https://127.0.0.1:45652/wxiz/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting 
response from server at: https://127.0.0.1:45652/wxiz/collection1
        at 
__randomizedtesting.SeedInfo.seed([E76CDD475AEFCD4D:6F38E29DF413A0B5]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:451)
        at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:392)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1383)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1134)
        at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1073)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
        at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1535)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.docValuesUpdateTest(TestInPlaceUpdatesDistrib.java:391)
        at 
org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:160)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
        at sun.security.ssl.InputRecord.read(InputRecord.java:503)
        at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
        at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
        at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:532)
        ... 54 more


FAILED:  
junit.framework.TestSuite.org.apache.solr.update.TestInPlaceUpdatesDistrib

Error Message:
7 threads leaked from SUITE scope at 
org.apache.solr.update.TestInPlaceUpdatesDistrib:     1) Thread[id=134, 
name=qtp1986118145-134, state=RUNNABLE, group=TGRP-TestInPlaceUpdatesDistrib]   
      at java.util.HashMap.resize(HashMap.java:734)         at 
java.util.HashMap.putVal(HashMap.java:662)         at 
java.util.HashMap.put(HashMap.java:611)         at 
java.util.HashSet.add(HashSet.java:219)         at 
java.util.AbstractCollection.addAll(AbstractCollection.java:344)         at 
org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)         at 
org.apache.lucene.index.IndexFileDeleter$CommitPoint.<init>(IndexFileDeleter.java:789)
         at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:524)  
       at 
org.apache.lucene.index.IndexWriter.finishCommit(IndexWriter.java:3297)         
at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3254)    
     at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3209)       
  at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:682)
         at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
         at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
         at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
         at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=97, 
name=searcherExecutor-31-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=162, 
name=searcherExecutor-53-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=76, 
name=qtp1174328245-76, state=RUNNABLE, group=TGRP-TestInPlaceUpdatesDistrib]    
     at 
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
         at 
org.apache.lucene.index.AssertingLeafReader$AssertingTermsEnum.seekExact(AssertingLeafReader.java:257)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDocValuesUpdates(BufferedUpdatesStream.java:667)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDocValuesUpdatesList(BufferedUpdatesStream.java:613)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:269)
         at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
         at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)    
     at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3421)      
   at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3378)         
at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1125)         
at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1170)         at 
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:280)         
at 
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:898)
         at 
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
         at 
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:392)
         at 
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:81) 
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1539)         at 
org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:581)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:375)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=101, 
name=qtp1013664760-101, state=BLOCKED, group=TGRP-TestInPlaceUpdatesDistrib]    
     at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4867) 
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
         at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
         at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235) 
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)    
     at 
org.apache.solr.update.UpdateLog.openRealtimeSearcher(UpdateLog.java:638)       
  at 
org.apache.solr.update.IndexFingerprint.getFingerprint(IndexFingerprint.java:94)
         at 
org.apache.solr.handler.component.RealTimeGetComponent.processGetFingeprint(RealTimeGetComponent.java:947)
         at 
org.apache.solr.handler.component.RealTimeGetComponent.process(RealTimeGetComponent.java:123)
         at 
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=106, 
name=qtp1013664760-106, state=RUNNABLE, group=TGRP-TestInPlaceUpdatesDistrib]   
      at java.util.HashSet.<init>(HashSet.java:118)         at 
org.apache.lucene.index.SegmentCommitInfo.files(SegmentCommitInfo.java:220)     
    at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)        
 at org.apache.lucene.index.IndexFileDeleter.incRef(IndexFileDeleter.java:553)  
       at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:520)  
       at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2481) 
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:651)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:272)
         at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
         at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)    
     at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3050)
         at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3246)       
  at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3209)         
at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:682)
         at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
         at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
         at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
         at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=125, 
name=searcherExecutor-42-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE 
scope at org.apache.solr.update.TestInPlaceUpdatesDistrib: 
   1) Thread[id=134, name=qtp1986118145-134, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at java.util.HashMap.resize(HashMap.java:734)
        at java.util.HashMap.putVal(HashMap.java:662)
        at java.util.HashMap.put(HashMap.java:611)
        at java.util.HashSet.add(HashSet.java:219)
        at java.util.AbstractCollection.addAll(AbstractCollection.java:344)
        at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)
        at 
org.apache.lucene.index.IndexFileDeleter$CommitPoint.<init>(IndexFileDeleter.java:789)
        at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:524)
        at 
org.apache.lucene.index.IndexWriter.finishCommit(IndexWriter.java:3297)
        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3254)
        at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3209)
        at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:682)
        at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
        at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=97, name=searcherExecutor-31-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=162, name=searcherExecutor-53-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=76, name=qtp1174328245-76, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at 
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
        at 
org.apache.lucene.index.AssertingLeafReader$AssertingTermsEnum.seekExact(AssertingLeafReader.java:257)
        at 
org.apache.lucene.index.BufferedUpdatesStream.applyDocValuesUpdates(BufferedUpdatesStream.java:667)
        at 
org.apache.lucene.index.BufferedUpdatesStream.applyDocValuesUpdatesList(BufferedUpdatesStream.java:613)
        at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:269)
        at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
        at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)
        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3421)
        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3378)
        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1125)
        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1170)
        at 
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:280)
        at 
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:898)
        at 
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at 
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:392)
        at 
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:81)
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1539)
        at org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:581)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:375)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=101, name=qtp1013664760-101, state=BLOCKED, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at 
org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4867)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
        at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)
        at 
org.apache.solr.update.UpdateLog.openRealtimeSearcher(UpdateLog.java:638)
        at 
org.apache.solr.update.IndexFingerprint.getFingerprint(IndexFingerprint.java:94)
        at 
org.apache.solr.handler.component.RealTimeGetComponent.processGetFingeprint(RealTimeGetComponent.java:947)
        at 
org.apache.solr.handler.component.RealTimeGetComponent.process(RealTimeGetComponent.java:123)
        at 
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=106, name=qtp1013664760-106, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at java.util.HashSet.<init>(HashSet.java:118)
        at 
org.apache.lucene.index.SegmentCommitInfo.files(SegmentCommitInfo.java:220)
        at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)
        at 
org.apache.lucene.index.IndexFileDeleter.incRef(IndexFileDeleter.java:553)
        at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:520)
        at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2481)
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:651)
        at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:272)
        at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
        at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)
        at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3050)
        at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3246)
        at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3209)
        at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:682)
        at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
        at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=125, name=searcherExecutor-42-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
        at __randomizedtesting.SeedInfo.seed([E76CDD475AEFCD4D]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.update.TestInPlaceUpdatesDistrib

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=134, name=qtp1986118145-134, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]         at 
org.apache.lucene.index.IndexWriter$ReaderPool.noDups(IndexWriter.java:714)     
    at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:703) 
        at 
org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
         at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:467) 
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
         at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
         at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235) 
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)    
     at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2199)         
at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1936)         at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:713)
         at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
         at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
         at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
         at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=97, 
name=searcherExecutor-31-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=162, 
name=searcherExecutor-53-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=76, 
name=qtp1174328245-76, state=RUNNABLE, group=TGRP-TestInPlaceUpdatesDistrib]    
     at java.util.HashMap.newNode(HashMap.java:1742)         at 
java.util.HashMap.putVal(HashMap.java:641)         at 
java.util.HashMap.put(HashMap.java:611)         at 
java.util.HashSet.add(HashSet.java:219)         at 
java.util.AbstractCollection.addAll(AbstractCollection.java:344)         at 
org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)         at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:540)  
       at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2481) 
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:651)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:272)
         at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
         at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)    
     at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3421)      
   at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3378)         
at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1125)         
at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1170)         at 
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:280)         
at 
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:898)
         at 
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
         at 
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:392)
         at 
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:81) 
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1539)         at 
org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:581)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:375)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=101, 
name=qtp1013664760-101, state=BLOCKED, group=TGRP-TestInPlaceUpdatesDistrib]    
     at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4867) 
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
         at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
         at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235) 
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)    
     at 
org.apache.solr.update.UpdateLog.openRealtimeSearcher(UpdateLog.java:638)       
  at 
org.apache.solr.update.IndexFingerprint.getFingerprint(IndexFingerprint.java:94)
         at 
org.apache.solr.handler.component.RealTimeGetComponent.processGetFingeprint(RealTimeGetComponent.java:947)
         at 
org.apache.solr.handler.component.RealTimeGetComponent.process(RealTimeGetComponent.java:123)
         at 
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:295)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=106, 
name=qtp1013664760-106, state=RUNNABLE, group=TGRP-TestInPlaceUpdatesDistrib]   
      at java.lang.Thread.holdsLock(Native Method)         at 
org.apache.lucene.index.IndexFileDeleter.locked(IndexFileDeleter.java:117)      
   at 
org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:630)      
   at 
org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:583)      
   at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:534)  
       at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2481) 
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:651)
         at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:272)
         at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
         at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)    
     at 
org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3050)
         at 
org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3246)       
  at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3209)         
at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:682)
         at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
         at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
         at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
         at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)         
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)         
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)      
   at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)       
  at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)   
      at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)  
       at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
        at org.eclipse.jetty.server.Server.handle(Server.java:534)         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)     
    at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)       
  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
         at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)    
     at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=125, 
name=searcherExecutor-42-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=134, name=qtp1986118145-134, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at 
org.apache.lucene.index.IndexWriter$ReaderPool.noDups(IndexWriter.java:714)
        at 
org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:703)
        at 
org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
        at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:467)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:293)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
        at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2199)
        at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1936)
        at 
org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:713)
        at 
org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93)
        at 
org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1937)
        at 
org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1913)
        at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160)
        at 
org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:68)
        at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62)
        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:178)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2487)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:362)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=97, name=searcherExecutor-31-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=162, name=searcherExecutor-53-thread-1, state=WAITING, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=76, name=qtp1174328245-76, state=RUNNABLE, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at java.util.HashMap.newNode(HashMap.java:1742)
        at java.util.HashMap.putVal(HashMap.java:641)
        at java.util.HashMap.put(HashMap.java:611)
        at java.util.HashSet.add(HashSet.java:219)
        at java.util.AbstractCollection.addAll(AbstractCollection.java:344)
        at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:792)
        at 
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:540)
        at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2481)
        at 
org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:651)
        at 
org.apache.lucene.index.BufferedUpdatesStream.applyDeletesAndUpdates(BufferedUpdatesStream.java:272)
        at 
org.apache.lucene.index.IndexWriter.applyAllDeletesAndUpdates(IndexWriter.java:3458)
        at 
org.apache.lucene.index.IndexWriter.maybeApplyDeletes(IndexWriter.java:3444)
        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3421)
        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3378)
        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1125)
        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1170)
        at 
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:280)
        at 
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:898)
        at 
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at 
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:392)
        at 
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:81)
        at org.apache.solr.core.SolrCore.close(SolrCore.java:1539)
        at org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:581)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:375)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:534)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=101, name=qtp1013664760-101, state=BLOCKED, 
group=TGRP-TestInPlaceUpdatesDistrib]
        at 
org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4867)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:289)
        at 
org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:278)
        at 
org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:235)
        at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1990)
        at org.apache.solr.update.UpdateLog.openRealtimeSe

[...truncated too long message...]

t org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]    >         at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    >         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]    >         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]    >         at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    >         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]    >         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    >         at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]    >         at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    >         at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]    >         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]    >         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]    >         at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    >         at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    >         at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
   [junit4]    >         at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    >         at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    >         at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=125, name=searcherExecutor-42-thread-1, 
state=WAITING, group=TGRP-TestInPlaceUpdatesDistrib]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([E76CDD475AEFCD4D]:0)
   [junit4] Completed [375/709 (2!)] on J2 in 3693.38s, 1 test, 3 errors <<< 
FAILURES!

[...truncated 49 lines...]
   [junit4] Suite: org.apache.solr.TestGroupingSearch
   [junit4] ERROR   0.00s J2 | TestGroupingSearch.initializationError <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit 
exceeded
   [junit4]    >        at java.util.Arrays.copyOfRange(Arrays.java:3664)
   [junit4]    >        at java.lang.String.<init>(String.java:207)
   [junit4]    >        at 
java.lang.StringBuilder.toString(StringBuilder.java:407)
   [junit4]    >        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:187)
   [junit4]    >        at 
java.lang.reflect.Method.toGenericString(Method.java:415)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
   [junit4]    >        at java.util.TimSort.binarySort(TimSort.java:296)
   [junit4]    >        at java.util.TimSort.sort(TimSort.java:239)
   [junit4]    >        at java.util.Arrays.sort(Arrays.java:1438)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
   [junit4]    >        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4] Completed [392/709 (3!)] on J2 in 0.38s, 1 test, 1 error <<< 
FAILURES!

[...truncated 4 lines...]
   [junit4] Suite: org.apache.solr.handler.admin.ShowFileRequestHandlerTest
   [junit4] ERROR   0.50s J2 | ShowFileRequestHandlerTest.initializationError 
<<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit 
exceeded
   [junit4]    >        at 
java.lang.reflect.Modifier.toString(Modifier.java:235)
   [junit4]    >        at 
java.lang.reflect.Executable.printModifiersIfNonzero(Executable.java:100)
   [junit4]    >        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:146)
   [junit4]    >        at 
java.lang.reflect.Method.toGenericString(Method.java:415)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
   [junit4]    >        at java.util.TimSort.binarySort(TimSort.java:296)
   [junit4]    >        at java.util.TimSort.sort(TimSort.java:239)
   [junit4]    >        at java.util.Arrays.sort(Arrays.java:1438)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
   [junit4]    >        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4] Completed [394/709 (4!)] on J2 in 0.50s, 1 test, 1 error <<< 
FAILURES!

[...truncated 4 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
   [junit4] ERROR   0.00s J2 | TestSegmentSorting.initializationError <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit 
exceeded
   [junit4]    >        at java.util.Arrays.copyOf(Arrays.java:3332)
   [junit4]    >        at 
java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
   [junit4]    >        at 
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
   [junit4]    >        at 
java.lang.StringBuilder.append(StringBuilder.java:136)
   [junit4]    >        at 
java.lang.reflect.Executable.sharedToGenericString(Executable.java:171)
   [junit4]    >        at 
java.lang.reflect.Method.toGenericString(Method.java:415)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:26)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$1.compare(ClassModel.java:23)
   [junit4]    >        at java.util.TimSort.binarySort(TimSort.java:296)
   [junit4]    >        at java.util.TimSort.sort(TimSort.java:239)
   [junit4]    >        at java.util.Arrays.sort(Arrays.java:1438)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:216)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$3.members(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel$ModelBuilder.build(ClassModel.java:85)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.methodsModel(ClassModel.java:212)
   [junit4]    >        at 
com.carrotsearch.randomizedtesting.ClassModel.<init>(ClassModel.java:207)
   [junit4]    >        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4] Completed [396/709 (5!)] on J2 in 0.00s, 1 test, 1 error <<< 
FAILURES!

[...truncated 970 lines...]
   [junit4] JVM J2: stdout was not empty, see: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20170505_113721_5573700493991963119845.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid361.hprof
 ...
   [junit4] Heap dump file created [669120947 bytes in 4.178 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 7457 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:783:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:727:
 Some of the tests produced a heap dump, but did not fail. Maybe a suppressed 
OutOfMemoryError? Dumps created:
* java_pid361.hprof

Total time: 269 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org