Hi, Can you please try increasing stalltime by adding the below parameter in the startup script.
*-Dsolr.cloud.client.stallTime=60000* Also try to analyze the indexing load on the solr nodes and see if you can reduce it. On Wed, Apr 17, 2024 at 1:58 PM Stoyanov, Nikolay <nikolay.stoya...@docuware.com.invalid> wrote: > Hello, > I am using solr 9.1.1 in cloud mode with two replicas on different VMs. > > > > Lately we are experiencing issue with task queue processing. > > I have read about the other similar issues and tried the mentioned > solutions but nothing managed to solve the issue. > > Could you please assist me with what could be causing the issue and how to > solve it. > > > > Below you can find details from the exception. > > java.io.IOException: Task queue processing has stalled for 20191 ms with 0 > remaining elements to process. > at > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.blockUntilFinished(ConcurrentUpdateHttp2SolrClient.java:508) > at > org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:87) > at > org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:292) > at > org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:91) > at > org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1263) > at > org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1253) > at > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > at > org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:94) > at > org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224) > at org.apache.solr.core.SolrCore.execute(SolrCore.java:2865) > at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887) > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606) > at > org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:250) > at > org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:218) > at > org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257) > at > org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227) > at > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:213) > at > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:195) > at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201) > at > org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1626) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:552) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) > at > org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) > at > org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:505) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) > at > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191) > at > org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177) > at > org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at > org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) > at > org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:772) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at org.eclipse.jetty.server.Server.handle(Server.java:516) > at > org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) > at org.eclipse.jetty.io > .AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) > at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) > at > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) > at java.base/java.lang.Thread.run(Thread.java:840) > > Thank you in advance! > > Best Regards, > > > Nikolay Stoyanov > Cloud SysOps - Nemetschek OOD > > www.nemetschek.bg<http://www.nemetschek.bg> | e-mail: > nstoya...@nemetschek.bg | office: +359-2-4210-900 | mobile: > +359-887-262-838 > > ________________________________ > > DISCLAIMER: Confidential. Dissemination, distribution or copying of this > e-mail is prohibited. If you have received this e-mail in error, please > notify the sender by replying to this message and delete it immediately. > Nemetschek OOD shall not be liable for the message if altered, changed or > falsified. > > P Please consider the impact on the environment before printing this e-mail > > > -- Thanks and Regards, Hari Mobile:9790756568
