Hi.
We installed a solr 10 test cluster.
We seem to run into a "header size" issue when query-ing our imported data.
We use 3 sets of node roles.
*
Coordinator
*
Data
*
importer
The idea is that we query the coordinator roles, and requests a forwarded to
the data nodes where the data is stored.
I think the header size limit is reached when the coordinator forwards its
reqeust to the data nodes,
We have already override values in place for the jetty.xml
javaOpts: "-Dsolr.node.roles=coordinator:off,data:on,overseer:disallowed
-Dsolr.otel.disabled=true -Dsolr.log.dir=/var/solr/data/logs
-Dlog4j2.formatMsgNoLookups=true -Dsecurity.overridePropertiesFile=true
-Dsolr.allowPaths=/backups -Dsolr.jetty.request.header.size=65535
-Dsolr.jetty.response.header.size=65535 -Dsolr.http.requestHeaderSize=65535
-Dsolr.http.responseHeaderSize=65535 -Dsolr.jetty.output.aggregation.size=32767
"
I also added this to the jetty-http.xml
<Call name="addConnector">
<Arg>
<New class="org.eclipse.jetty.server.ServerConnector">
<Arg name="server"><Ref refid="Server" /></Arg>
<Arg name="acceptors" type="int"><Property
name="solr.jetty.http.acceptors" default="-1"/></Arg>
<Arg name="selectors" type="int"><Property
name="solr.jetty.http.selectors" default="-1"/></Arg>
<Arg name="factories">
<Array type="org.eclipse.jetty.server.ConnectionFactory">
<Item>
<New class="org.eclipse.jetty.server.HttpConnectionFactory">
<Arg name="config"><Ref refid="httpConfig" /></Arg>
</New>
</Item>
<Item>
<New
class="org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory">
<Arg name="config"><Ref refid="httpConfig" /></Arg>
<Set name="maxDecoderTableCapacity">16384</Set>
<Set name="maxEncoderTableCapacity">16384</Set>
</New>
</Item>
</Array>
</Arg>
This is the complete error message
{
"error":{
"metadata":{
"error-class":"org.apache.solr.common.SolrException",
"root-error-class":"org.eclipse.jetty.http2.hpack.HpackException$SessionException"
},
"errorClass":"org.apache.solr.common.SolrException",
"msg":"Error trying to proxy request for url:
http://solr-importer-solrcloud-0.solr-importer-solrcloud-headless.netm-solr-searchv2.svc.cluster.local:8983/solr/packages_cnl_acc
with _forwardCount: 1",
"trace":{
"stackTrace":[
"org.apache.solr.servlet.HttpSolrCall.sendRemoteProxy(HttpSolrCall.java:651)",
"org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:470)",
"org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:227)",
"org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilterRetry$0(SolrDispatchFilter.java:186)",
"org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:226)",
"org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:194)",
"org.apache.solr.servlet.SolrDispatchFilter.doFilterRetry(SolrDispatchFilter.java:180)",
"org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:170)",
"jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)",
"org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:208)",
"org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1594)",
"org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1555)",
"org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:823)",
"org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:440)",
"org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)",
"org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:575)",
"org.eclipse.jetty.ee10.servlet.SessionHandler.handle(SessionHandler.java:717)",
"org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1071)",
"org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:151)",
"org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)",
"org.eclipse.jetty.server.handler.ConditionalHandler.nextHandler(ConditionalHandler.java:421)",
"org.eclipse.jetty.server.handler.InetAccessHandler.onConditionsMet(InetAccessHandler.java:50)",
"org.eclipse.jetty.server.handler.ConditionalHandler.handle(ConditionalHandler.java:378)",
"org.eclipse.jetty.server.Handler$Sequence.handle(Handler.java:805)",
"org.eclipse.jetty.rewrite.handler.RewriteHandler$LastRuleHandler.handle(RewriteHandler.java:159)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:143)",
"org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:611)",
"org.eclipse.jetty.server.Server.handle(Server.java:182)",
"org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:677)",
"org.eclipse.jetty.server.internal.HttpConnection.onFillable(HttpConnection.java:416)",
"org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:322)",
"org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)",
"org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:480)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:443)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:293)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:201)",
"org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:311)",
"org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)",
"org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1211)",
"org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)",
"java.base/java.lang.Thread.run(Unknown Source)"
],
"causedBy":{
"errorClass":"org.eclipse.jetty.http2.hpack.HpackException$SessionException",
"msg":"Header size 9002 > 8192",
"trace":{
"stackTrace":[
"org.eclipse.jetty.http2.hpack.HpackEncoder.encode(HpackEncoder.java:288)",
"org.eclipse.jetty.http2.generator.FrameGenerator.encode(FrameGenerator.java:64)",
"org.eclipse.jetty.http2.generator.HeadersGenerator.generateHeaders(HeadersGenerator.java:67)",
"org.eclipse.jetty.http2.generator.HeadersGenerator.generate(HeadersGenerator.java:53)",
"org.eclipse.jetty.http2.generator.Generator.control(Generator.java:88)",
"org.eclipse.jetty.http2.HTTP2Session$ControlEntry.generate(HTTP2Session.java:1635)",
"org.eclipse.jetty.http2.internal.HTTP2Flusher.process(HTTP2Flusher.java:243)",
"org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:253)",
"org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:232)",
"org.eclipse.jetty.http2.HTTP2Session$StreamsState.flush(HTTP2Session.java:2744)",
"org.eclipse.jetty.http2.HTTP2Session$StreamsState.createLocalStream(HTTP2Session.java:2541)",
"org.eclipse.jetty.http2.HTTP2Session$StreamsState.newLocalStream(HTTP2Session.java:2457)",
"org.eclipse.jetty.http2.HTTP2Session.newStream(HTTP2Session.java:765)",
"org.eclipse.jetty.http2.client.transport.internal.HttpSenderOverHTTP2.sendHeaders(HttpSenderOverHTTP2.java:126)",
"org.eclipse.jetty.client.transport.HttpSender$ContentSender.process(HttpSender.java:574)",
"org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:253)",
"org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:232)",
"org.eclipse.jetty.client.transport.HttpSender.send(HttpSender.java:85)",
"org.eclipse.jetty.http2.client.transport.internal.HttpChannelOverHTTP2.send(HttpChannelOverHTTP2.java:112)",
"org.eclipse.jetty.client.transport.HttpChannel.send(HttpChannel.java:142)",
"org.eclipse.jetty.client.transport.HttpConnection.send(HttpConnection.java:113)",
"org.eclipse.jetty.http2.client.transport.internal.HttpConnectionOverHTTP2.send(HttpConnectionOverHTTP2.java:153)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:448)",
"org.eclipse.jetty.client.transport.HttpDestination.process(HttpDestination.java:421)",
"org.eclipse.jetty.client.transport.HttpDestination.process(HttpDestination.java:375)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:358)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:352)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:329)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:308)",
"org.eclipse.jetty.client.transport.HttpRequest.sendAsync(HttpRequest.java:758)",
"org.eclipse.jetty.client.transport.HttpDestination.send(HttpDestination.java:303)",
"org.eclipse.jetty.client.transport.HttpRequest.send(HttpRequest.java:751)",
"org.apache.solr.servlet.HttpSolrProxy.doHttpProxy(HttpSolrProxy.java:80)",
"org.apache.solr.servlet.HttpSolrCall.sendRemoteProxy(HttpSolrCall.java:648)",
"org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:470)",
"org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:227)",
"org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilterRetry$0(SolrDispatchFilter.java:186)",
"org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:226)",
"org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:194)",
"org.apache.solr.servlet.SolrDispatchFilter.doFilterRetry(SolrDispatchFilter.java:180)",
"org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:170)",
"jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)",
"org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:208)",
"org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1594)",
"org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1555)",
"org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:823)",
"org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:440)",
"org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)",
"org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:575)",
"org.eclipse.jetty.ee10.servlet.SessionHandler.handle(SessionHandler.java:717)",
"org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1071)",
"org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:151)",
"org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)",
"org.eclipse.jetty.server.handler.ConditionalHandler.nextHandler(ConditionalHandler.java:421)",
"org.eclipse.jetty.server.handler.InetAccessHandler.onConditionsMet(InetAccessHandler.java:50)",
"org.eclipse.jetty.server.handler.ConditionalHandler.handle(ConditionalHandler.java:378)",
"org.eclipse.jetty.server.Handler$Sequence.handle(Handler.java:805)",
"org.eclipse.jetty.rewrite.handler.RewriteHandler$LastRuleHandler.handle(RewriteHandler.java:159)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.HeaderPatternRule$1.handle(HeaderPatternRule.java:89)",
"org.eclipse.jetty.rewrite.handler.Rule$Handler.handle(Rule.java:108)",
"org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:143)",
"org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:611)",
"org.eclipse.jetty.server.Server.handle(Server.java:182)",
"org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:677)",
"org.eclipse.jetty.server.internal.HttpConnection.onFillable(HttpConnection.java:416)",
"org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:322)",
"org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)",
"org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:480)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:443)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:293)",
"org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:201)",
"org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:311)",
"org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)",
"org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1211)",
"org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)",
"java.base/java.lang.Thread.run(Unknown Source)"
]
}
}
},
"code":500
}
}
Anny ideas???
