[
https://issues.apache.org/jira/browse/HTTPASYNC-153?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16912042#comment-16912042
]
Max Rozhkov commented on HTTPASYNC-153:
---------------------------------------
# Httpcore is 4.4.11
# There are 40 threads. Below several more stack traces with corresponding
monitor ids
# We've got a high loaded ssp server. The issue could reproduce after an hour
or several hours of work. No idea how to make a snippet for it :)
# Thank you for answering. I just wanted to let you know that the situation is
critical. We are considering all possibilities. Hope there is a way to fix the
problem.
"I/O dispatcher 15" #98 prio=5 os_prio=0 cpu=77512.81ms elapsed=3719.80s
tid=0x00007f7cc0094800 nid=0x639a runnable [0x00007f7d21ed0000]
java.lang.Thread.State: RUNNABLE
at
sun.security.ssl.SSLEngineImpl.getHandshakeStatus([email protected]/SSLEngineImpl.java:805)
- locked <0x00000011c1c7d060> (a sun.security.ssl.SSLEngineImpl)
at
org.apache.http.nio.reactor.ssl.SSLIOSession.doHandshake(SSLIOSession.java:293)
at
org.apache.http.nio.reactor.ssl.SSLIOSession.isAppInputReady(SSLIOSession.java:503)
- locked <0x00000011c1c7d080> (a org.apache.http.nio.reactor.ssl.SSLIOSession)
at
org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:120)
at
org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)
at
org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588)
at java.lang.Thread.run([email protected]/Thread.java:834)
"I/O dispatcher 14" #97 prio=5 os_prio=0 cpu=77515.92ms elapsed=3719.80s
tid=0x00007f7cc0092800 nid=0x6399 runnable [0x00007f7d21fd1000]
java.lang.Thread.State: RUNNABLE
at
sun.security.ssl.SSLEngineImpl.checkParams([email protected]/SSLEngineImpl.java:397)
at sun.security.ssl.SSLEngineImpl.wrap([email protected]/SSLEngineImpl.java:133)
- locked <0x00000011c1caa280> (a sun.security.ssl.SSLEngineImpl)
at sun.security.ssl.SSLEngineImpl.wrap([email protected]/SSLEngineImpl.java:116)
- locked <0x00000011c1caa280> (a sun.security.ssl.SSLEngineImpl)
at javax.net.ssl.SSLEngine.wrap([email protected]/SSLEngine.java:479)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doWrap(SSLIOSession.java:263)
at
org.apache.http.nio.reactor.ssl.SSLIOSession.doHandshake(SSLIOSession.java:301)
at
org.apache.http.nio.reactor.ssl.SSLIOSession.isAppInputReady(SSLIOSession.java:503)
- locked <0x00000011c1caa2a0> (a org.apache.http.nio.reactor.ssl.SSLIOSession)
at
org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:120)
at
org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)
at
org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)
at
org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588)
at java.lang.Thread.run([email protected]/Thread.java:834)
> Deadlock when sending new request
> ---------------------------------
>
> Key: HTTPASYNC-153
> URL: https://issues.apache.org/jira/browse/HTTPASYNC-153
> Project: HttpComponents HttpAsyncClient
> Issue Type: Bug
> Affects Versions: 4.1.4
> Environment: CentOS Linux release 7.5.1804
> Java 11
> Reporter: Max Rozhkov
> Priority: Blocker
>
> We faced a situation that async http requests are blocked at the moment they
> are being made. This blocks work thread which make all the threads in the
> pool blocked.
> At the same time, the thread which closes idle connections is also blocked.
> Please fix the problem. Otherwise we have to move to another http client asap.
> Typical stack traces are below:
>
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.http.nio.reactor.ssl.SSLIOSession.close(SSLIOSession.java:605)
> - waiting to lock <0x00000011c1caa2a0> (a
> org.apache.http.nio.reactor.ssl.SSLIOSession)
> at
> org.apache.http.impl.nio.NHttpConnectionBase.close(NHttpConnectionBase.java:511)
> at
> org.apache.http.impl.nio.conn.CPoolEntry.closeConnection(CPoolEntry.java:75)
> at org.apache.http.impl.nio.conn.CPoolEntry.close(CPoolEntry.java:101)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool$4.process(AbstractNIOConnPool.java:846)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.enumAvailable(AbstractNIOConnPool.java:775)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.closeIdle(AbstractNIOConnPool.java:841)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.closeIdleConnections(PoolingNHttpClientConnectionManager.java:493)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.lambda$monitorConnections$0(AsyncHttpClient.java:67)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient$$Lambda$311/0x00000017c2616440.run(Unknown
> Source)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at
> java.util.concurrent.FutureTask.runAndReset([email protected]/FutureTask.java:305)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:305)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
>
> java.lang.Thread.State: WAITING (parking)
> at jdk.internal.misc.Unsafe.park([email protected]/Native Method)
> - parking to wait for <0x00000011310a4150> (a
> java.util.concurrent.locks.ReentrantLock$NonfairSync)
> at
> java.util.concurrent.locks.LockSupport.park([email protected]/LockSupport.java:194)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt([email protected]/AbstractQueuedSynchronizer.java:885)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued([email protected]/AbstractQueuedSynchronizer.java:917)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire([email protected]/AbstractQueuedSynchronizer.java:1240)
> at
> java.util.concurrent.locks.ReentrantLock.lock([email protected]/ReentrantLock.java:267)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:278)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:377)
> at
> org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:129)
> at
> org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:141)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:75)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:108)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.execute(AsyncHttpClient.java:96)
> at
> net.thumbtack.ssp.requester.SelfHealingAsyncHttpClient.execute(SelfHealingAsyncHttpClient.java:45)
> at
> net.thumbtack.ssp.requester.AsyncHttpRequester.get(AsyncHttpRequester.java:125)
> at net.thumbtack.ssp.requester.HttpRequester.get(HttpRequester.java:38)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.doWinNotification(PartnerHttpNotifier.java:138)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.notifyPartner(PartnerHttpNotifier.java:65)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$notifyPartner$1(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$1897/0x00000017c2a13040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.notifyPartner(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$getImpressionsLogger$0(NotificationEventLoggerFactory.java:39)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$324/0x00000017c2612c40.log(Unknown
> Source)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$8(CompositeAsyncEventLoggerFactory.java:109)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1889/0x00000017c2a15040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$9(CompositeAsyncEventLoggerFactory.java:107)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1881/0x00000017c2a17040.run(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture$AsyncRun.run([email protected]/CompletableFuture.java:1736)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:304)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
>
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.http.nio.reactor.ssl.SSLIOSession.setEvent(SSLIOSession.java:672)
> - waiting to lock <0x00000011c1c7d080> (a
> org.apache.http.nio.reactor.ssl.SSLIOSession)
> at
> org.apache.http.impl.nio.NHttpConnectionBase.requestOutput(NHttpConnectionBase.java:358)
> at
> org.apache.http.impl.nio.conn.CPoolProxy.requestOutput(CPoolProxy.java:120)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.connectionAllocated(AbstractClientExchangeHandler.java:334)
> - locked <0x00000011c1d046c0> (a org.apache.http.impl.nio.SessionHttpContext)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.access$000(AbstractClientExchangeHandler.java:62)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.completed(AbstractClientExchangeHandler.java:387)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler$1.completed(AbstractClientExchangeHandler.java:383)
> at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:122)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.completed(PoolingNHttpClientConnectionManager.java:306)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager$1.completed(PoolingNHttpClientConnectionManager.java:297)
> at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:122)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.fireCallbacks(AbstractNIOConnPool.java:506)
> at
> org.apache.http.nio.pool.AbstractNIOConnPool.lease(AbstractNIOConnPool.java:290)
> at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.requestConnection(PoolingNHttpClientConnectionManager.java:295)
> at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.requestConnection(AbstractClientExchangeHandler.java:377)
> at
> org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.start(DefaultClientExchangeHandlerImpl.java:129)
> at
> org.apache.http.impl.nio.client.InternalHttpAsyncClient.execute(InternalHttpAsyncClient.java:141)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:75)
> at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClient.execute(CloseableHttpAsyncClient.java:108)
> at
> net.thumbtack.ssp.requester.AsyncHttpClient.execute(AsyncHttpClient.java:96)
> at
> net.thumbtack.ssp.requester.SelfHealingAsyncHttpClient.execute(SelfHealingAsyncHttpClient.java:45)
> at
> net.thumbtack.ssp.requester.AsyncHttpRequester.get(AsyncHttpRequester.java:125)
> at net.thumbtack.ssp.requester.HttpRequester.get(HttpRequester.java:38)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.doWinNotification(PartnerHttpNotifier.java:138)
> at
> net.thumbtack.ssp.impression.tracking.util.PartnerHttpNotifier.notifyPartner(PartnerHttpNotifier.java:59)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.lambda$notifyPartner$1(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$1897/0x00000017c2a13040.accept(Unknown
> Source)
> at
> java.util.Collections$SingletonList.forEach([email protected]/Collections.java:4854)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory.notifyPartner(NotificationEventLoggerFactory.java:43)
> at
> net.thumbtack.adtech.notification.NotificationEventLoggerFactory$$Lambda$323/0x00000017c2612840.log(Unknown
> Source)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$8(CompositeAsyncEventLoggerFactory.java:109)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1889/0x00000017c2a15040.accept(Unknown
> Source)
> at java.util.ArrayList.forEach([email protected]/ArrayList.java:1540)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory.lambda$runLoggersAsync$9(CompositeAsyncEventLoggerFactory.java:107)
> at
> net.thumbtack.adtech.event.CompositeAsyncEventLoggerFactory$$Lambda$1881/0x00000017c2a17040.run(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture$AsyncRun.run([email protected]/CompletableFuture.java:1736)
> at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
> at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:304)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run([email protected]/Thread.java:834)
--
This message was sent by Atlassian Jira
(v8.3.2#803003)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
