[
https://issues.apache.org/jira/browse/HTTPASYNC-145?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Max Rozhkov updated HTTPASYNC-145:
----------------------------------
Description:
*User Story*
We use async http client in very high load environment where several thousands
of http requests are sent per second. It is an SSP service where timeout has to
be measured and a request to be voided after timeout.
We set up usual timeouts configured in IO Reactor and control timeouts
externally as well. When time is out the request will be cancelled explicitly
with _AbstractExecutionAwareRequest.abort()_.
*Problem description*
Deadlock could happen in very highload environment. See stack information below.
*Stack trace*
{{"timeout-controller-0-00":}}
{{ at
java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:108)}}
{{ - waiting to lock <0x000000112c420bd0> (a java.lang.Object)}}
{{ at
org.apache.http.impl.nio.reactor.IOSessionImpl.close(IOSessionImpl.java:227)}}
{{ - locked <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)}}
{{ at
org.apache.http.impl.nio.reactor.IOSessionImpl.shutdown(IOSessionImpl.java:255)}}
{{ at
org.apache.http.impl.nio.NHttpConnectionBase.shutdown(NHttpConnectionBase.java:579)}}
{{ at
org.apache.http.impl.nio.conn.CPoolEntry.shutdownConnection(CPoolEntry.java:80)}}
{{ at org.apache.http.impl.nio.conn.CPoolProxy.shutdown(CPoolProxy.java:91)}}
{{ at
org.apache.http.impl.nio.client.AbstractClientExchangeHandler.discardConnection(AbstractClientExchangeHandler.java:267)}}
{{ at
org.apache.http.impl.nio.client.AbstractClientExchangeHandler.cancel(AbstractClientExchangeHandler.java:447)}}
{{ at
org.apache.http.client.methods.AbstractExecutionAwareRequest.abort(AbstractExecutionAwareRequest.java:90)}}
{{ at
net.thumbtack.ssp.requester.AsyncHttpClient.lambda$execute$1(AsyncHttpClient.java:83)}}
{{ at
net.thumbtack.ssp.requester.AsyncHttpClient$$Lambda$1768/0x00000017c29f6440.run(Unknown
Source)}}
{{ at
net.thumbtack.adtech.concurrent.TimeLimitedExecution.lambda$future$1(TimeLimitedExecution.java:70)}}
{{ at
net.thumbtack.adtech.concurrent.TimeLimitedExecution$$Lambda$1770/0x00000017c29f6c40.run(Unknown
Source)}}
{{ at
net.thumbtack.adtech.concurrent.TimeoutController.check(TimeoutController.java:47)}}
{{ at
net.thumbtack.adtech.concurrent.TimeoutController$$Lambda$1022/0x00000017c28acc40.run(Unknown
Source)}}
{{ at
java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)}}
{{ at
java.util.concurrent.FutureTask.runAndReset([email protected]/FutureTask.java:305)}}
{{ at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:305)}}
{{ at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)}}
{{ at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)}}
{{ at java.lang.Thread.run([email protected]/Thread.java:834)}}
{{"pool-21-thread-1":}}
{{ at
java.nio.channels.spi.AbstractSelectionKey.cancel([email protected]/AbstractSelectionKey.java:70)}}
{{ - waiting to lock <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)}}
{{ at
java.nio.channels.spi.AbstractSelectableChannel.implCloseChannel([email protected]/AbstractSelectableChannel.java:255)}}
{{ at
java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:112)}}
{{ - locked <0x000000112c420bd0> (a java.lang.Object)}}
{{ at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.doShutdown(AbstractMultiworkerIOReactor.java:414)}}
{{ at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:374)}}
{{ at
org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221)}}
{{ at
org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)}}
{{ at java.lang.Thread.run([email protected]/Thread.java:834) }}
was:
*User Story*
We use async http client in very high load environment where several thousands
of http requests are sent per second. It is an SSP service where timeout has to
be measured and a request to be voided after timeout.
We set up usual timeouts configured in IO Reactor and control timeouts
externally as well. When time is out the request will be cancelled explicitly
with _AbstractExecutionAwareRequest.abort()_.
*Problem description*
Deadlock could happen in very highload environment. See stack information below.
*Stack trace*
"timeout-controller-0-00":
at
java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:108)
- waiting to lock <0x000000112c420bd0> (a java.lang.Object)
at org.apache.http.impl.nio.reactor.IOSessionImpl.close(IOSessionImpl.java:227)
- locked <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)
at
org.apache.http.impl.nio.reactor.IOSessionImpl.shutdown(IOSessionImpl.java:255)
at
org.apache.http.impl.nio.NHttpConnectionBase.shutdown(NHttpConnectionBase.java:579)
at
org.apache.http.impl.nio.conn.CPoolEntry.shutdownConnection(CPoolEntry.java:80)
at org.apache.http.impl.nio.conn.CPoolProxy.shutdown(CPoolProxy.java:91)
at
org.apache.http.impl.nio.client.AbstractClientExchangeHandler.discardConnection(AbstractClientExchangeHandler.java:267)
at
org.apache.http.impl.nio.client.AbstractClientExchangeHandler.cancel(AbstractClientExchangeHandler.java:447)
at
org.apache.http.client.methods.AbstractExecutionAwareRequest.abort(AbstractExecutionAwareRequest.java:90)
at
net.thumbtack.ssp.requester.AsyncHttpClient.lambda$execute$1(AsyncHttpClient.java:83)
at
net.thumbtack.ssp.requester.AsyncHttpClient$$Lambda$1768/0x00000017c29f6440.run(Unknown
Source)
at
net.thumbtack.adtech.concurrent.TimeLimitedExecution.lambda$future$1(TimeLimitedExecution.java:70)
at
net.thumbtack.adtech.concurrent.TimeLimitedExecution$$Lambda$1770/0x00000017c29f6c40.run(Unknown
Source)
at
net.thumbtack.adtech.concurrent.TimeoutController.check(TimeoutController.java:47)
at
net.thumbtack.adtech.concurrent.TimeoutController$$Lambda$1022/0x00000017c28acc40.run(Unknown
Source)
at
java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)
at
java.util.concurrent.FutureTask.runAndReset([email protected]/FutureTask.java:305)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:305)
at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)
at java.lang.Thread.run([email protected]/Thread.java:834)
"pool-21-thread-1":
at
java.nio.channels.spi.AbstractSelectionKey.cancel([email protected]/AbstractSelectionKey.java:70)
- waiting to lock <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)
at
java.nio.channels.spi.AbstractSelectableChannel.implCloseChannel([email protected]/AbstractSelectableChannel.java:255)
at
java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:112)
- locked <0x000000112c420bd0> (a java.lang.Object)
at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.doShutdown(AbstractMultiworkerIOReactor.java:414)
at
org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:374)
at
org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221)
at
org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)
at java.lang.Thread.run([email protected]/Thread.java:834)
> Eliminate a deadlock in IOReactor
> ---------------------------------
>
> Key: HTTPASYNC-145
> URL: https://issues.apache.org/jira/browse/HTTPASYNC-145
> Project: HttpComponents HttpAsyncClient
> Issue Type: Bug
> Affects Versions: 4.1.4
> Reporter: Max Rozhkov
> Priority: Major
>
> *User Story*
> We use async http client in very high load environment where several
> thousands of http requests are sent per second. It is an SSP service where
> timeout has to be measured and a request to be voided after timeout.
> We set up usual timeouts configured in IO Reactor and control timeouts
> externally as well. When time is out the request will be cancelled explicitly
> with _AbstractExecutionAwareRequest.abort()_.
> *Problem description*
> Deadlock could happen in very highload environment. See stack information
> below.
> *Stack trace*
> {{"timeout-controller-0-00":}}
> {{ at
> java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:108)}}
> {{ - waiting to lock <0x000000112c420bd0> (a java.lang.Object)}}
> {{ at
> org.apache.http.impl.nio.reactor.IOSessionImpl.close(IOSessionImpl.java:227)}}
> {{ - locked <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)}}
> {{ at
> org.apache.http.impl.nio.reactor.IOSessionImpl.shutdown(IOSessionImpl.java:255)}}
> {{ at
> org.apache.http.impl.nio.NHttpConnectionBase.shutdown(NHttpConnectionBase.java:579)}}
> {{ at
> org.apache.http.impl.nio.conn.CPoolEntry.shutdownConnection(CPoolEntry.java:80)}}
> {{ at org.apache.http.impl.nio.conn.CPoolProxy.shutdown(CPoolProxy.java:91)}}
> {{ at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.discardConnection(AbstractClientExchangeHandler.java:267)}}
> {{ at
> org.apache.http.impl.nio.client.AbstractClientExchangeHandler.cancel(AbstractClientExchangeHandler.java:447)}}
> {{ at
> org.apache.http.client.methods.AbstractExecutionAwareRequest.abort(AbstractExecutionAwareRequest.java:90)}}
> {{ at
> net.thumbtack.ssp.requester.AsyncHttpClient.lambda$execute$1(AsyncHttpClient.java:83)}}
> {{ at
> net.thumbtack.ssp.requester.AsyncHttpClient$$Lambda$1768/0x00000017c29f6440.run(Unknown
> Source)}}
> {{ at
> net.thumbtack.adtech.concurrent.TimeLimitedExecution.lambda$future$1(TimeLimitedExecution.java:70)}}
> {{ at
> net.thumbtack.adtech.concurrent.TimeLimitedExecution$$Lambda$1770/0x00000017c29f6c40.run(Unknown
> Source)}}
> {{ at
> net.thumbtack.adtech.concurrent.TimeoutController.check(TimeoutController.java:47)}}
> {{ at
> net.thumbtack.adtech.concurrent.TimeoutController$$Lambda$1022/0x00000017c28acc40.run(Unknown
> Source)}}
> {{ at
> java.util.concurrent.Executors$RunnableAdapter.call([email protected]/Executors.java:515)}}
> {{ at
> java.util.concurrent.FutureTask.runAndReset([email protected]/FutureTask.java:305)}}
> {{ at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run([email protected]/ScheduledThreadPoolExecutor.java:305)}}
> {{ at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1128)}}
> {{ at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:628)}}
> {{ at java.lang.Thread.run([email protected]/Thread.java:834)}}
> {{"pool-21-thread-1":}}
> {{ at
> java.nio.channels.spi.AbstractSelectionKey.cancel([email protected]/AbstractSelectionKey.java:70)}}
> {{ - waiting to lock <0x000000112c420be0> (a sun.nio.ch.SelectionKeyImpl)}}
> {{ at
> java.nio.channels.spi.AbstractSelectableChannel.implCloseChannel([email protected]/AbstractSelectableChannel.java:255)}}
> {{ at
> java.nio.channels.spi.AbstractInterruptibleChannel.close([email protected]/AbstractInterruptibleChannel.java:112)}}
> {{ - locked <0x000000112c420bd0> (a java.lang.Object)}}
> {{ at
> org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.doShutdown(AbstractMultiworkerIOReactor.java:414)}}
> {{ at
> org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:374)}}
> {{ at
> org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221)}}
> {{ at
> org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)}}
> {{ at java.lang.Thread.run([email protected]/Thread.java:834) }}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
