I get a "500 Exception" error log message at regular intervals after upgrading from 9.4.0 to 9.5.0. I'm using SolrCloud.
Here's the full exception: org.apache.solr.common.SolrException: org.apache.solr.client.solrj.SolrServerException at org.apache.solr.handler.component.SearchHandler.throwSolrException(SearchHandler.java:665) at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:583) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:226) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2884) at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:875) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:561) at org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:262) at org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:219) at org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:249) at org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:215) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:213) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:195) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:210) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:598) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1553) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:149) at org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:228) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:822) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.Server.handle(Server.java:563) at org.eclipse.jetty.server.HttpChannel$RequestDispatchable.dispatch(HttpChannel.java:1598) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:753) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:501) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:287) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100) at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:199) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149) at java.base/java.lang.Thread.run(Unknown Source) Caused by: org.apache.solr.client.solrj.SolrServerException at org.apache.solr.client.solrj.impl.Http2SolrClient$1.onFailure(Http2SolrClient.java:542) at org.eclipse.jetty.client.ResponseNotifier.notifyFailure(ResponseNotifier.java:192) at org.eclipse.jetty.client.ResponseNotifier.notifyFailure(ResponseNotifier.java:184) at org.eclipse.jetty.client.HttpReceiver.abort(HttpReceiver.java:559) at org.eclipse.jetty.client.HttpChannel.abortResponse(HttpChannel.java:154) at org.eclipse.jetty.client.HttpChannel.abort(HttpChannel.java:147) at org.eclipse.jetty.client.HttpExchange.abort(HttpExchange.java:273) at org.eclipse.jetty.client.HttpConversation.abort(HttpConversation.java:159) at org.eclipse.jetty.client.HttpRequest.abort(HttpRequest.java:925) at org.eclipse.jetty.http2.client.http.HttpConnectionOverHTTP2.abort(HttpConnectionOverHTTP2.java:253) at org.eclipse.jetty.http2.client.http.HttpConnectionOverHTTP2.close(HttpConnectionOverHTTP2.java:226) at org.eclipse.jetty.http2.client.http.HttpConnectionOverHTTP2.close(HttpConnectionOverHTTP2.java:217) at org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2.onClose(HttpClientTransportOverHTTP2.java:175) at org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2$SessionListenerPromise.onClose(HttpClientTransportOverHTTP2.java:194) at org.eclipse.jetty.http2.client.http.HTTPSessionListenerPromise.onClose(HTTPSessionListenerPromise.java:101) at org.eclipse.jetty.http2.api.Session$Listener.onClose(Session.java:260) at org.eclipse.jetty.http2.HTTP2Session.notifyClose(HTTP2Session.java:1197) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.terminate(HTTP2Session.java:2141) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.lambda$sendGoAwayAndTerminate$16(HTTP2Session.java:2047) at org.eclipse.jetty.util.Callback$6.completed(Callback.java:223) at org.eclipse.jetty.util.Callback$Nested.failed(Callback.java:409) at org.eclipse.jetty.util.Callback$Nested.failed(Callback.java:405) at org.eclipse.jetty.http2.HTTP2Flusher$Entry.failed(HTTP2Flusher.java:459) at org.eclipse.jetty.http2.HTTP2Flusher.lambda$onCompleteFailure$1(HTTP2Flusher.java:373) at java.base/java.lang.Iterable.forEach(Unknown Source) at org.eclipse.jetty.http2.HTTP2Flusher.onCompleteFailure(HTTP2Flusher.java:373) at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:328) at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:224) at org.eclipse.jetty.http2.HTTP2Session.frame(HTTP2Session.java:821) at org.eclipse.jetty.http2.HTTP2Session.frames(HTTP2Session.java:793) at org.eclipse.jetty.http2.HTTP2Session.control(HTTP2Session.java:775) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.sendGoAway(HTTP2Session.java:2052) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.sendGoAwayAndTerminate(HTTP2Session.java:2047) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.lambda$onGoAway$4(HTTP2Session.java:1763) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.tryRunZeroStreamsAction(HTTP2Session.java:2123) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.onStreamDestroyed(HTTP2Session.java:2067) at org.eclipse.jetty.http2.HTTP2Session.onStreamDestroyed(HTTP2Session.java:1094) at org.eclipse.jetty.http2.HTTP2Session.removeStream(HTTP2Session.java:932) at org.eclipse.jetty.http2.HTTP2Stream.onFailure(HTTP2Stream.java:588) at org.eclipse.jetty.http2.HTTP2Stream.process(HTTP2Stream.java:407) at org.eclipse.jetty.http2.HTTP2Session.failStream(HTTP2Session.java:628) at org.eclipse.jetty.http2.HTTP2Session.failStreams(HTTP2Session.java:620) at org.eclipse.jetty.http2.HTTP2Session$StreamsState.onGoAway(HTTP2Session.java:1843) at org.eclipse.jetty.http2.HTTP2Session.onGoAway(HTTP2Session.java:498) at org.eclipse.jetty.http2.HTTP2Connection.onGoAway(HTTP2Connection.java:284) at org.eclipse.jetty.http2.parser.BodyParser.notifyGoAway(BodyParser.java:187) at org.eclipse.jetty.http2.parser.GoAwayBodyParser.onGoAway(GoAwayBodyParser.java:167) at org.eclipse.jetty.http2.parser.GoAwayBodyParser.parse(GoAwayBodyParser.java:137) at org.eclipse.jetty.http2.parser.Parser.parseBody(Parser.java:240) at org.eclipse.jetty.http2.parser.Parser.parse(Parser.java:167) at org.eclipse.jetty.http2.HTTP2Connection$HTTP2Producer.produce(HTTP2Connection.java:350) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:455) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:248) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:193) at org.eclipse.jetty.http2.HTTP2Connection.produce(HTTP2Connection.java:208) at org.eclipse.jetty.http2.HTTP2Connection.onFillable(HTTP2Connection.java:155) at org.eclipse.jetty.http2.HTTP2Connection$FillableCallback.succeeded(HTTP2Connection.java:450) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100) at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53) at org.eclipse.jetty.util.thread.Invocable.invokeNonBlocking(Invocable.java:151) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.invokeAsNonBlocking(AdaptiveExecutionStrategy.java:438) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:380) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:193) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:299) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ... 1 more Caused by: java.nio.channels.AsynchronousCloseException ... 57 more And some examples of the regular intervalled errors (notice that all end at second :06): 26.2.2024, 05:37:06 ERROR false foo_offers_shard7_replica_n32 HttpSolrCall 500 Exception 26.2.2024, 06:22:06 ERROR false foo_offers_shard10_replica_n6 HttpSolrCall 500 Exception 26.2.2024, 06:35:06 ERROR false foo_offers_shard15_replica_n22 HttpSolrCall 500 Exception 26.2.2024, 06:47:06 ERROR false foo_offers_shard15_replica_n22 HttpSolrCall 500 Exception 26.2.2024, 07:35:06 ERROR false foo_offers_shard10_replica_n6 HttpSolrCall 500 Exception 26.2.2024, 07:46:06 ERROR false foo_offers_shard15_replica_n22 HttpSolrCall 500 Exception 26.2.2024, 08:05:06 ERROR false foo_offers_shard15_replica_n22 HttpSolrCall 500 Exception 26.2.2024, 08:46:06 ERROR false foo_offers_shard7_replica_n32 HttpSolrCall 500 Exception 26.2.2024, 09:40:06 ERROR false foo_offers_shard7_replica_n32 HttpSolrCall 500 Exception Also reported here: https://issues.apache.org/jira/browse/SOLR-17177 Any ideas? Cheers, Henrik
