Which version of Solr is it? On Mon, 19 Feb, 2024, 1:21 pm HariBabu kuruva, <[email protected]> wrote:
> Hi All, > > Am facing the same error in the PROD environment. > > It would be very helpful if someone helps to figure out the issue. > > On Tue, May 16, 2023 at 8:45 AM Mingchun Zhao <[email protected]> > wrote: > > > Hello, > > > > We have recently seen the below errors repeated when update requests were > > received by Solr. Could you please advise on how to deal with this issue. > > The error and warning from the Solr log as below: > > > > ======================================== > > 2023-05-09 10:03:05.128 ERROR > > (updateExecutor-8-thread-11857-processing-test_shard1_replica_n1 > core_node3 > > null stg-test-search--1.i.kandasearch.com:8983_solr test shard1) [c:test > > s:shard1 r: > > > > core_node3 x:test_shard1_replica_n1] > > o.a.s.u.ErrorReportingConcurrentUpdateSolrClient Error when calling > > SolrCmdDistributor$Req: cmd=add{,id=xxx,commitWithin=30000}; > > < > > > https://www.toyota.co.jp/jpn/company/history/75years/text/leaping_forward_as_a_global_corporation/chapter2/section1/item3.html,commitWithin=30000%7D > > ;> > > node=ForwardNode: http://test.com:8983/solr/test_shard1_replica_n2/ > > < > > > http://stg-toyota-search--2.i.kandasearch.com:8983/solr/test-mcf_shard1_replica_n2/ > > > > > to http://test.com:8983/solr/test_shard1_replica_n2/ > > < > > > http://stg-toyota-search--2.i.kandasearch.com:8983/solr/test-mcf_shard1_replica_n2/ > > > > > => java.io.IOException: java.util.concurrent.TimeoutException: Total > > timeout 600000 ms elapsed > > at > > > > > org.eclipse.jetty.client.util.DeferredContentProvider.flush(DeferredContentProvider.java:197) > > java.io.IOException: java.util.concurrent.TimeoutException: Total timeout > > 600000 ms elapsed > > at > > > > > org.eclipse.jetty.client.util.DeferredContentProvider.flush(DeferredContentProvider.java:197) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.util.OutputStreamContentProvider$DeferredOutputStream.flush(OutputStreamContentProvider.java:151) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.util.OutputStreamContentProvider$DeferredOutputStream.write(OutputStreamContentProvider.java:145) > > ~[?:?] > > at > > > > > org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:207) > > ~[?:?] > > at > > > > > org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:200) > > ~[?:?] > > at > org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:169) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:100) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.BinaryRequestWriter.write(BinaryRequestWriter.java:80) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.Http2SolrClient.send(Http2SolrClient.java:367) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient$Runner.sendUpdateStream(ConcurrentUpdateHttp2SolrClient.java:238) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient$Runner.run(ConcurrentUpdateHttp2SolrClient.java:180) > > ~[?:?] > > at > > > > > com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) > > ~[metrics-core-4.1.5.jar:4.1.5] > > at > > > > > org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271) > > ~[?:?] > > at > > > > > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > > ~[?:?] > > at > > > > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > > ~[?:?] > > at java.lang.Thread.run(Thread.java:829) ~[?:?] > > Suppressed: java.io.IOException: java.util.concurrent.TimeoutException: > > Total timeout 600000 ms elapsed > > at > > > > > org.eclipse.jetty.client.util.DeferredContentProvider.flush(DeferredContentProvider.java:197) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.util.OutputStreamContentProvider$DeferredOutputStream.flush(OutputStreamContentProvider.java:151) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.util.OutputStreamContentProvider$DeferredOutputStream.write(OutputStreamContentProvider.java:145) > > ~[?:?] > > at > > > > > org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:207) > > ~[?:?] > > at > > > > > org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:200) > > ~[?:?] > > at org.apache.solr.common.util.JavaBinCodec.close(JavaBinCodec.java:1286) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:99) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.BinaryRequestWriter.write(BinaryRequestWriter.java:80) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.Http2SolrClient.send(Http2SolrClient.java:367) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient$Runner.sendUpdateStream(ConcurrentUpdateHttp2SolrClient.java:238) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient$Runner.run(ConcurrentUpdateHttp2SolrClient.java:180) > > ~[?:?] > > at > > > > > com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) > > ~[metrics-core-4.1.5.jar:4.1.5] > > at > > > > > org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271) > > ~[?:?] > > at > > > > > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > > ~[?:?] > > at > > > > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > > ~[?:?] > > at java.lang.Thread.run(Thread.java:829) ~[?:?] > > Caused by: java.util.concurrent.TimeoutException: Total timeout 600000 ms > > elapsed > > at > > > > > org.eclipse.jetty.client.HttpDestination$RequestTimeouts.onExpired(HttpDestination.java:567) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.HttpDestination$RequestTimeouts.onExpired(HttpDestination.java:550) > > ~[?:?] > > at > > org.eclipse.jetty.io > > .CyclicTimeouts.onTimeoutExpired(CyclicTimeouts.java:110) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io > .CyclicTimeouts.access$100(CyclicTimeouts.java:50) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > org.eclipse.jetty.io > > .CyclicTimeouts$Timeouts.onTimeoutExpired(CyclicTimeouts.java:196) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io.CyclicTimeout$Wakeup.run(CyclicTimeout.java:298) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) > > ~[?:?] > > at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] > > at > > > > > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) > > ~[?:?] > > ... 3 more > > Caused by: java.util.concurrent.TimeoutException: Total timeout 600000 ms > > elapsed > > at > > > > > org.eclipse.jetty.client.HttpDestination$RequestTimeouts.onExpired(HttpDestination.java:567) > > ~[?:?] > > at > > > > > org.eclipse.jetty.client.HttpDestination$RequestTimeouts.onExpired(HttpDestination.java:550) > > ~[?:?] > > at > > org.eclipse.jetty.io > > .CyclicTimeouts.onTimeoutExpired(CyclicTimeouts.java:110) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io > .CyclicTimeouts.access$100(CyclicTimeouts.java:50) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > org.eclipse.jetty.io > > .CyclicTimeouts$Timeouts.onTimeoutExpired(CyclicTimeouts.java:196) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io.CyclicTimeout$Wakeup.run(CyclicTimeout.java:298) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) > > ~[?:?] > > at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] > > at > > > > > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) > > ~[?:?] > > ... 3 more > > 2023-05-09 10:03:05.396 WARN (qtp1167088639-27634) [c:test s:shard1 r: > > core_node3 x:test_shard1_replica_n1] o.a.s.u.SolrCmdDistributor Unable to > > finish sending updates => java.io.IOException: Task queue processing has > > stalled for 20008 ms with 1 remaining elements to process. > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.waitForEmptyQueue(ConcurrentUpdateHttp2SolrClient.java:602) > > java.io.IOException: Task queue processing has stalled for 20008 ms with > 1 > > remaining elements to process. > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.waitForEmptyQueue(ConcurrentUpdateHttp2SolrClient.java:602) > > ~[?:?] > > at > > > > > org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.blockUntilFinished(ConcurrentUpdateHttp2SolrClient.java:475) > > ~[?:?] > > at > > > > > org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:87) > > ~[?:?] > > at > > > > > org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:292) > > ~[?:?] > > at > > > > > org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:91) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1263) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1253) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:197) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > com.rondhuit.solr.update.KnownWordsExtractUpdateProcessorFactory$KnownWordsExtractUpdateProcessor.finish(KnownWordsExtractUpdateProcessorFactory.java:240) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:78) > > ~[?:?] > > at > > > > > org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:94) > > ~[?:?] > > at > > > > > org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224) > > ~[?:?] > > at org.apache.solr.core.SolrCore.execute(SolrCore.java:2865) ~[?:?] > > at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887) > > ~[?:?] > > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606) > ~[?:?] > > at > > > > > org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:250) > > ~[?:?] > > at > > > > > org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:218) > > ~[?:?] > > at > > > > > org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257) > > ~[?:?] > > at > > > > > org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227) > > ~[?:?] > > at > > > > > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:213) > > ~[?:?] > > at > > > > > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:195) > > ~[?:?] > > at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201) > > ~[jetty-servlet-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1626) > > ~[jetty-servlet-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:552) > > ~[jetty-servlet-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) > > ~[jetty-security-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:505) > > ~[jetty-servlet-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) > > ~[jetty-rewrite-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:772) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.server.Server.handle(Server.java:516) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) > > ~[jetty-server-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > org.eclipse.jetty.io > > .AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) > > ~[jetty-io-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at > > > > > org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) > > ~[jetty-util-9.4.48.v20220622.jar:9.4.48.v20220622] > > at java.lang.Thread.run(Thread.java:829) ~[?:?] > > ======================================== > > > > Kind regards, > > Mingchun > > > > > -- > > Thanks and Regards, > Hari > Mobile:9790756568 >
