The issue popped up again with the LAX policy.
ERROR 2025-09-18T23:48:32.505409235Z [resource.labels.containerName: worker] Exception in thread "idle-connection-evictor-1" java.lang.StackOverflowError ERROR 2025-09-18T23:48:32.506882480Z [resource.labels.containerName: worker] at java.base/java.lang.Character.toLowerCase(Character.java:10965) ERROR 2025-09-18T23:48:32.506911615Z [resource.labels.containerName: worker] at java.base/java.lang.Character.toLowerCase(Character.java:10936) ERROR 2025-09-18T23:48:32.506915519Z [resource.labels.containerName: worker] at java.base/sun.net.util.IPAddressUtil.parseAsciiHexDigit(IPAddressUtil.java:843) ERROR 2025-09-18T23:48:32.506918031Z [resource.labels.containerName: worker] at java.base/sun.net.util.IPAddressUtil.parseAsciiDigit(IPAddressUtil.java:835) ERROR 2025-09-18T23:48:32.506920408Z [resource.labels.containerName: worker] at java.base/sun.net.util.IPAddressUtil.digit(IPAddressUtil.java:637) ERROR 2025-09-18T23:48:32.506922809Z [resource.labels.containerName: worker] at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1635) ERROR 2025-09-18T23:48:32.506925821Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:61) ERROR 2025-09-18T23:48:32.506928465Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.DnsResolver.resolve(DnsResolver.java:77) ERROR 2025-09-18T23:48:32.506931691Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.MultihomeIOSessionRequester.connect(MultihomeIOSessionRequester.java:113) ERROR 2025-09-18T23:48:32.506936477Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.DefaultAsyncClientConnectionOperator.connect(DefaultAsyncClientConnectionOperator.java:115) ERROR 2025-09-18T23:48:32.506938974Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.connect(PoolingAsyncClientConnectionManager.java:456) ERROR 2025-09-18T23:48:32.506941414Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.connectEndpoint(InternalHttpAsyncExecRuntime.java:226) ERROR 2025-09-18T23:48:32.506943914Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.doProceedToNextHop(AsyncConnectExec.java:222) ERROR 2025-09-18T23:48:32.506946173Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:197) ERROR 2025-09-18T23:48:32.506948477Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.506950903Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.506953202Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.506955599Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.506958036Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.506960414Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.506962924Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.506965328Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.506979486Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.506982134Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.506984809Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.506987274Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.506989633Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.507004103Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.507006539Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.507008861Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.507011234Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.507013693Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.507016004Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.507018296Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.507020660Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.507023059Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.507025394Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.507027766Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.507030207Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.507032891Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.507038636Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.507056042Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.507066928Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507069499Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.507071897Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.507074420Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.507077389Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507080176Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.507082549Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.507084946Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.507092153Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.507094467Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.507096863Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.507099148Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.507101480Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.507103920Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.507106285Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.507108600Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.507111153Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.507113447Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.507115759Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.507118328Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.507120582Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.507122894Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.507125332Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.507127714Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507130058Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.507132535Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.507134838Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.507137170Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507139474Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.507141717Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.507143970Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.507146302Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.507148579Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.507150921Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.507153219Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.507157858Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.507164546Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.507166961Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.507169286Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.507171673Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.507174012Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.507176339Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.507178653Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.507180975Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.507183301Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.507185589Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.507187912Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507190318Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.507192596Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.507194893Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.507196916Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.507199021Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.507200972Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.507203143Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.507205366Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.507207617Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.507210093Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.507212481Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.507214697Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.507217052Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.507392861Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.507414032Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.509192879Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.509204224Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.509213630Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.509215976Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.509218210Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.509220697Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.509223106Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.509225848Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509228189Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.509230489Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.509232659Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.509234722Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509236871Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.509239030Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.509241156Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.509243281Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.509245374Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.509247574Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.509249460Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.509251205Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.509320687Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.509377749Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.509380456Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.509382756Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.509384893Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.509386960Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.509389144Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.509391340Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.509393426Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.509395532Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.509449128Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509451840Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.509453939Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.509456154Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.509458855Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509460965Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.509549323Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.509552362Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.509554435Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.509556578Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.509558724Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.509560776Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.509562849Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.509564949Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.509567008Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.509569221Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.509571439Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.509573670Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.509575731Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.509577470Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.509579162Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.509624994Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.509709501Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.509711778Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509713828Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.509716293Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.509718710Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.509725383Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509727586Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.509729746Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.509731842Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.509733946Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.509736129Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.509809398Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.509851740Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.509854537Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.509856767Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.509858979Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.509861176Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.509863411Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.509865400Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.509867592Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.509869735Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.509871835Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.509873986Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.509876161Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.509878210Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509880519Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.509882685Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.509884896Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.509886980Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.509889163Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.509891318Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.509893497Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.509895899Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.509898077Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.510058763Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.510070845Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.510073129Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.510075288Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.510077384Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.510079488Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.510081710Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.510083992Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.510086129Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.510088253Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.510090716Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.510092866Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.510094940Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.510097109Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.510249941Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.510253732Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.510255967Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.510258184Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.510260412Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.510262682Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.510264827Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.510266978Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.510269451Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.510271625Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.510275677Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.510277909Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.510280100Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.510286988Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.510289230Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.510291328Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.510293738Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.510295996Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.510298155Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.513213997Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.513225178Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.513227876Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.513230125Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.513232288Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-09-18T23:48:32.513234541Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-09-18T23:48:32.513236747Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-09-18T23:48:32.513238736Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-09-18T23:48:32.513240918Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$LeaseRequest.completed(LaxConnPool.java:334) ERROR 2025-09-18T23:48:32.513243038Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequests(LaxConnPool.java:544) ERROR 2025-09-18T23:48:32.513245257Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.servicePendingRequest(LaxConnPool.java:522) ERROR 2025-09-18T23:48:32.513247374Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool$PerRoutePool.release(LaxConnPool.java:517) ERROR 2025-09-18T23:48:32.513249721Z [resource.labels.containerName: worker] at org.apache.hc.core5.pool.LaxConnPool.release(LaxConnPool.java:181) ERROR 2025-09-18T23:48:32.513252166Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-09-18T23:48:32.513254494Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-09-18T23:48:32.513256766Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-09-18T23:48:32.513259370Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-09-18T23:48:32.513261787Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-09-18T23:48:32.513285173Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-09-18T23:48:32.513287982Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-09-18T23:48:32.513292437Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:199) ERROR 2025-09-18T23:48:32.513295020Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-09-18T23:48:32.513305833Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-09-18T23:48:32.513308423Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-09-18T23:48:32.513317716Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-09-18T23:48:32.513320200Z [resource.labels.containerName: worker] at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-09-18T23:48:32.513322633Z [resource.labels.containerName: worker] at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) On 19. Sep 2025, at 09:02, Stephan Epping <[email protected]> wrote: Hello, I have investigated a tricky error we experienced multiple times in our benchmarks (that use the httpclient to send thousands of requests to our clusters). We are using a quite up-to-date version (5.3.4 / 5.5.). You can find more details about the analysis here<https://github.com/camunda/camunda/issues/34597#issuecomment-3301797932>. TLDR; The stacktrace shows a tight synchronous callback cycle inside HttpComponents' async path that repeatedly alternates between completed → release/discard/fail → connect/proceedToNextHop → completed, causing unbounded recursion until the JVM stack overflows. Concretely the cycle is: •• AsyncConnectExec$1.completed → InternalHttpAsyncExecRuntime$1.completed → BasicFuture.completed •• PoolingAsyncClientConnectionManager lease/completed → StrictConnPool.fireCallbacks → StrictConnPool.release → PoolingAsyncClientConnectionManager.release •• InternalHttpAsyncExecRuntime.discardEndpoint → InternalAbstractHttpAsyncClient$2.failed → AsyncRedirectExec/AsyncHttpRequestRetryExec/AsyncProtocolExec/AsyncConnectExec.*failed → BasicFuture.failed / ComplexFuture.failed → PoolingAsyncClientConnectionManager$4.failed → DefaultAsyncClientConnectionOperator$1.failed → MultihomeIOSessionRequester.connect → DefaultAsyncClientConnectionOperator.connect → PoolingAsyncClientConnectionManager.connect → InternalHttpAsyncExecRuntime.connectEndpoint → AsyncConnectExec.proceedToNextHop → back to AsyncConnectExec$1.completed. Because callbacks (completed / failed) are executed synchronously on the same call stack and some code paths both complete and then trigger failed/retry/next-hop connection logic (via pool callbacks and the connection operator), the call stack never unwinds — recursion depth grows until StackOverflowError. Possible concrete root causes (detailed) 1. Synchronous BasicFuture callbacks BasicFuture.completed() and .failed() call callbacks immediately on the thread that completes the future. If a callback in turn calls pool release() which calls fireCallbacks() (synchronously), the chain can re-enter callback code without unwinding. Re-entrancy depth grows with each attempted connect/release cycle. 2. Multihome connect tries multiple addresses in the same stack MultihomeIOSessionRequester.connect will attempt alternate addresses (A/AAAA records). If an address fails quickly and the code immediately tries the next address by invoking connection manager code and its callbacks synchronously, you build deeper recursion for each try. 3. Retries/redirects executed synchronously The exec chain (redirect → retry → protocol → connect) will call failed() listeners which in turn call connect again. If those calls are synchronous, you get direct recursive invocation. 4. Potential omission of an async boundary A simple but dangerous pattern is: complete future → call listener → listener calls code that completes other futures → repeat. If there is no executor handoff, the recursion remains on the same thread. I haven’t been able to create a unit test that reproduces the issue locally, even though I tried multiple approaches (synthetic http server that is flaky, randomly failing custom dns resolver, thousands of requests scheduled, etc.). Does someone have an idea what we are doing wrong? Is this a bug or misconfiguration on our side? We switched now to the `LAX` concurrency policy which seems to mitigate the issue, but I believe it’s not fixing the root cause, but makes it less likely. (I can see the lax pool also has the sync fireCallbacks approach etc.) I have attached a stacktrace, but here a brief excerpt (as I don’t know if attachments work in this mailing list): ERROR 2025-06-30T17:06:10.233570881Z Exception in thread "httpclient-dispatch-1" java.lang.StackOverflowError -------------------------------------------------------------------------------- ERROR 2025-06-30T17:06:10.299219300Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-06-30T17:06:10.299221745Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-06-30T17:06:10.299223952Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-06-30T17:06:10.299226128Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-06-30T17:06:10.299228287Z at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-06-30T17:06:10.299230488Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-06-30T17:06:10.299232722Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-06-30T17:06:10.299234969Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-06-30T17:06:10.299237136Z at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-06-30T17:06:10.299239404Z at org.apache.hc.core5.pool.StrictConnPool.fireCallbacks(StrictConnPool.java:401) ERROR 2025-06-30T17:06:10.299241531Z at org.apache.hc.core5.pool.StrictConnPool.release(StrictConnPool.java:272) ERROR 2025-06-30T17:06:10.299243647Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-06-30T17:06:10.299245815Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-06-30T17:06:10.299247909Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-06-30T17:06:10.299250099Z at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-06-30T17:06:10.299252352Z at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-06-30T17:06:10.299254470Z at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-06-30T17:06:10.299256671Z at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-06-30T17:06:10.299258827Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$2.failed(AsyncConnectExec.java:235) ERROR 2025-06-30T17:06:10.299261062Z at org.apache.hc.core5.concurrent.CallbackContribution.failed(CallbackContribution.java:52) ERROR 2025-06-30T17:06:10.299263164Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299265335Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299267498Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$4.failed(PoolingAsyncClientConnectionManager.java:485) ERROR 2025-06-30T17:06:10.299273172Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299275385Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299277443Z at org.apache.hc.client5.http.impl.nio.DefaultAsyncClientConnectionOperator$1.failed(DefaultAsyncClientConnectionOperator.java:170) ERROR 2025-06-30T17:06:10.299279661Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299281730Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299283917Z at org.apache.hc.client5.http.impl.nio.MultihomeIOSessionRequester.connect(MultihomeIOSessionRequester.java:118) ERROR 2025-06-30T17:06:10.299287550Z at org.apache.hc.client5.http.impl.nio.DefaultAsyncClientConnectionOperator.connect(DefaultAsyncClientConnectionOperator.java:115) ERROR 2025-06-30T17:06:10.299290061Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.connect(PoolingAsyncClientConnectionManager.java:456) ERROR 2025-06-30T17:06:10.299292128Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.connectEndpoint(InternalHttpAsyncExecRuntime.java:226) ERROR 2025-06-30T17:06:10.299294317Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec.doProceedToNextHop(AsyncConnectExec.java:222) ERROR 2025-06-30T17:06:10.299296347Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec.proceedToNextHop(AsyncConnectExec.java:197) ERROR 2025-06-30T17:06:10.299298506Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec.access$000(AsyncConnectExec.java:92) ERROR 2025-06-30T17:06:10.299388975Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:164) ERROR 2025-06-30T17:06:10.299401176Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$1.completed(AsyncConnectExec.java:153) ERROR 2025-06-30T17:06:10.299403967Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:128) ERROR 2025-06-30T17:06:10.299406043Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime$1.completed(InternalHttpAsyncExecRuntime.java:120) ERROR 2025-06-30T17:06:10.299408227Z at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-06-30T17:06:10.299410406Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.leaseCompleted(PoolingAsyncClientConnectionManager.java:339) ERROR 2025-06-30T17:06:10.299412507Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:324) ERROR 2025-06-30T17:06:10.299414382Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$3$1.completed(PoolingAsyncClientConnectionManager.java:285) ERROR 2025-06-30T17:06:10.299426976Z at org.apache.hc.core5.concurrent.BasicFuture.completed(BasicFuture.java:148) ERROR 2025-06-30T17:06:10.299429367Z at org.apache.hc.core5.pool.StrictConnPool.fireCallbacks(StrictConnPool.java:401) ERROR 2025-06-30T17:06:10.299431117Z at org.apache.hc.core5.pool.StrictConnPool.release(StrictConnPool.java:272) ERROR 2025-06-30T17:06:10.299433157Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager.release(PoolingAsyncClientConnectionManager.java:424) ERROR 2025-06-30T17:06:10.299435324Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:156) ERROR 2025-06-30T17:06:10.299437225Z at org.apache.hc.client5.http.impl.async.InternalHttpAsyncExecRuntime.discardEndpoint(InternalHttpAsyncExecRuntime.java:180) ERROR 2025-06-30T17:06:10.299439134Z at org.apache.hc.client5.http.impl.async.InternalAbstractHttpAsyncClient$2.failed(InternalAbstractHttpAsyncClient.java:363) ERROR 2025-06-30T17:06:10.299451280Z at org.apache.hc.client5.http.impl.async.AsyncRedirectExec$1.failed(AsyncRedirectExec.java:261) ERROR 2025-06-30T17:06:10.299453403Z at org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec$1.failed(AsyncHttpRequestRetryExec.java:195) ERROR 2025-06-30T17:06:10.299455096Z at org.apache.hc.client5.http.impl.async.AsyncProtocolExec$1.failed(AsyncProtocolExec.java:297) ERROR 2025-06-30T17:06:10.299456747Z at org.apache.hc.client5.http.impl.async.AsyncConnectExec$2.failed(AsyncConnectExec.java:235) ERROR 2025-06-30T17:06:10.299458431Z at org.apache.hc.core5.concurrent.CallbackContribution.failed(CallbackContribution.java:52) ERROR 2025-06-30T17:06:10.299460343Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299462062Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299463899Z at org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager$4.failed(PoolingAsyncClientConnectionManager.java:485) ERROR 2025-06-30T17:06:10.299465810Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299467649Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299469389Z at org.apache.hc.client5.http.impl.nio.DefaultAsyncClientConnectionOperator$1.failed(DefaultAsyncClientConnectionOperator.java:170) ERROR 2025-06-30T17:06:10.299471284Z at org.apache.hc.core5.concurrent.BasicFuture.failed(BasicFuture.java:166) ERROR 2025-06-30T17:06:10.299473097Z at org.apache.hc.core5.concurrent.ComplexFuture.failed(ComplexFuture.java:79) ERROR 2025-06-30T17:06:10.299474928Z at org.apache.hc.client5.http.impl.nio.MultihomeIOSessionRequester.connect(MultihomeIOSessionRequester.java:118) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected]<mailto:[email protected]> For additional commands, e-mail: [email protected]<mailto:[email protected]>
