[
https://issues.apache.org/jira/browse/IGNITE-16136?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
David Albrecht updated IGNITE-16136:
------------------------------------
Attachment: image-2021-12-15-21-17-47-652.png
Description:
We are experiencing thread pool starvations and after some time out of memory
exceptions in some of our ignite client nodes while the server node seems to be
running without any problems. It seems like all sys threads are stuck when
calling MarshallerContextImpl.getClassName. Which in turn leads to a growing
worker queue.
{code:java}
10.12.21 11:22:34.603 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:27:34.654 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:32:34.713 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:37:34.764 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:42:34.796 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:47:34.839 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
{code}
{code}
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler
in thread "https-openssl-nio-16443-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler
in thread "ajp-nio-16009-ClientPoller"
11-Dec-2021 03:07:24.446 SEVERE [Catalina-utility-1]
org.apache.coyote.AbstractProtocol.startAsyncTimeout Error processing async
timeouts
java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError:
Java heap space
{code}
!image-2021-12-15-21-17-47-652.png!
It seems like all sys threads are stuck while waiting at:
{code}
class jdk.internal.misc.Unsafe @ 0xeaaf0ff0
class java.util.concurrent.locks.LockSupport @ 0xeaaf03f0
class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
class org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils
@ 0xef19cd38
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class org.apache.ignite.internal.GridEventConsumeHandler$EventWrapper @
0xef55b110
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class java.util.concurrent.ConcurrentLinkedDeque @ 0xeed74198
class jdk.internal.reflect.GeneratedMethodAccessor268 @ 0xece4a4d8
class jdk.internal.reflect.DelegatingMethodAccessorImpl @ 0xeb4f5ed0
class java.lang.reflect.Method @ 0xeae44f80
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller @
0xef30d3e0
class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @ 0xeeee22b0
class org.apache.ignite.internal.binary.BinaryUtils @ 0xef0d0fe0
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class
org.apache.ignite.internal.binary.BinaryFieldAccessor$DefaultFinalClassAccessor
@ 0xef2e7568
class org.apache.ignite.internal.binary.BinaryFieldAccessor @ 0xef304570
class org.apache.ignite.internal.binary.BinaryClassDescriptor @ 0xef30d458
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.GridBinaryMarshaller @ 0xef300f88
class org.apache.ignite.internal.binary.BinaryMarshaller @ 0xeefbdbc8
class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @ 0xeeee22b0
class org.apache.ignite.internal.util.IgniteUtils @ 0xeed806d0
class
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$7 @
0xef77ec78
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager$9 @
0xef6406e8
class java.util.concurrent.ThreadPoolExecutor @ 0xeab0fec0
class java.util.concurrent.ThreadPoolExecutor$Worker @ 0xeae473e8
class java.lang.Thread @ 0xeaaf0360
{code}
!image-2021-12-15-21-13-43-775.png!
was:
We are experiencing thread pool starvations and after some time out of memory
exceptions in some of our ignite client nodes while the server node seems to be
running without any problems.
{code:java}
10.12.21 11:22:34.603 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:27:34.654 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:32:34.713 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:37:34.764 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:42:34.796 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
10.12.21 11:47:34.839 [WARN ]
IgniteKernal.warning(127): Possible thread pool starvation detected (no task
completed in last 30000ms, is system thread pool size large enough?)
{code}
{code}
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler
in thread "https-openssl-nio-16443-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler
in thread "ajp-nio-16009-ClientPoller"
11-Dec-2021 03:07:24.446 SEVERE [Catalina-utility-1]
org.apache.coyote.AbstractProtocol.startAsyncTimeout Error processing async
timeouts
java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError:
Java heap space
{code}
It seems like all sys threads are stuck while waiting at:
{code}
class jdk.internal.misc.Unsafe @ 0xeaaf0ff0
class java.util.concurrent.locks.LockSupport @ 0xeaaf03f0
class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
class org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils
@ 0xef19cd38
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class org.apache.ignite.internal.GridEventConsumeHandler$EventWrapper @
0xef55b110
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class java.util.concurrent.ConcurrentLinkedDeque @ 0xeed74198
class jdk.internal.reflect.GeneratedMethodAccessor268 @ 0xece4a4d8
class jdk.internal.reflect.DelegatingMethodAccessorImpl @ 0xeb4f5ed0
class java.lang.reflect.Method @ 0xeae44f80
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor
@ 0xef665158
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class
org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
0xef77f128
class java.io.ObjectInputStream @ 0xeb61b050
class java.io.ObjectInputStream @ 0xeb61b050
class org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller @
0xef30d3e0
class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @ 0xeeee22b0
class org.apache.ignite.internal.binary.BinaryUtils @ 0xef0d0fe0
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class
org.apache.ignite.internal.binary.BinaryFieldAccessor$DefaultFinalClassAccessor
@ 0xef2e7568
class org.apache.ignite.internal.binary.BinaryFieldAccessor @ 0xef304570
class org.apache.ignite.internal.binary.BinaryClassDescriptor @ 0xef30d458
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
class org.apache.ignite.internal.binary.GridBinaryMarshaller @ 0xef300f88
class org.apache.ignite.internal.binary.BinaryMarshaller @ 0xeefbdbc8
class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @ 0xeeee22b0
class org.apache.ignite.internal.util.IgniteUtils @ 0xeed806d0
class
org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$7 @
0xef77ec78
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager @
0xef3304e0
class org.apache.ignite.internal.managers.communication.GridIoManager$9 @
0xef6406e8
class java.util.concurrent.ThreadPoolExecutor @ 0xeab0fec0
class java.util.concurrent.ThreadPoolExecutor$Worker @ 0xeae473e8
class java.lang.Thread @ 0xeaaf0360
{code}
!image-2021-12-15-21-13-43-775.png!
> System Thread pool starvation and out of memory
> -----------------------------------------------
>
> Key: IGNITE-16136
> URL: https://issues.apache.org/jira/browse/IGNITE-16136
> Project: Ignite
> Issue Type: Bug
> Affects Versions: 2.7.6
> Reporter: David Albrecht
> Priority: Critical
> Attachments: image-2021-12-15-21-13-43-775.png,
> image-2021-12-15-21-17-47-652.png
>
>
> We are experiencing thread pool starvations and after some time out of memory
> exceptions in some of our ignite client nodes while the server node seems to
> be running without any problems. It seems like all sys threads are stuck when
> calling MarshallerContextImpl.getClassName. Which in turn leads to a growing
> worker queue.
>
> {code:java}
> 10.12.21 11:22:34.603 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> 10.12.21 11:27:34.654 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> 10.12.21 11:32:34.713 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> 10.12.21 11:37:34.764 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> 10.12.21 11:42:34.796 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> 10.12.21 11:47:34.839 [WARN ]
> IgniteKernal.warning(127): Possible thread pool starvation detected (no task
> completed in last 30000ms, is system thread pool size large enough?)
> {code}
> {code}
> Exception: java.lang.OutOfMemoryError thrown from the
> UncaughtExceptionHandler in thread "https-openssl-nio-16443-ClientPoller"
> Exception: java.lang.OutOfMemoryError thrown from the
> UncaughtExceptionHandler in thread "ajp-nio-16009-ClientPoller"
> 11-Dec-2021 03:07:24.446 SEVERE [Catalina-utility-1]
> org.apache.coyote.AbstractProtocol.startAsyncTimeout Error processing async
> timeouts
> java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError:
> Java heap space
> {code}
> !image-2021-12-15-21-17-47-652.png!
> It seems like all sys threads are stuck while waiting at:
> {code}
> class jdk.internal.misc.Unsafe @ 0xeaaf0ff0
> class java.util.concurrent.locks.LockSupport @ 0xeaaf03f0
> class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
> class org.apache.ignite.internal.util.future.GridFutureAdapter @ 0xeee5f0b0
> class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
> class org.apache.ignite.internal.MarshallerContextImpl @ 0xef33c028
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils @
> 0xef19cd38
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class java.io.ObjectInputStream @ 0xeb61b050
> class java.io.ObjectInputStream @ 0xeb61b050
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor @
> 0xef665158
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class java.io.ObjectInputStream @ 0xeb61b050
> class java.io.ObjectInputStream @ 0xeb61b050
> class org.apache.ignite.internal.GridEventConsumeHandler$EventWrapper @
> 0xef55b110
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor @
> 0xef665158
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class java.io.ObjectInputStream @ 0xeb61b050
> class java.io.ObjectInputStream @ 0xeb61b050
> class java.util.concurrent.ConcurrentLinkedDeque @ 0xeed74198
> class jdk.internal.reflect.GeneratedMethodAccessor268 @ 0xece4a4d8
> class jdk.internal.reflect.DelegatingMethodAccessorImpl @ 0xeb4f5ed0
> class java.lang.reflect.Method @ 0xeae44f80
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedClassDescriptor @
> 0xef665158
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class
> org.apache.ignite.internal.marshaller.optimized.OptimizedObjectInputStream @
> 0xef77f128
> class java.io.ObjectInputStream @ 0xeb61b050
> class java.io.ObjectInputStream @ 0xeb61b050
> class org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller @
> 0xef30d3e0
> class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @
> 0xeeee22b0
> class org.apache.ignite.internal.binary.BinaryUtils @ 0xef0d0fe0
> class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
> class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
> class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
> class
> org.apache.ignite.internal.binary.BinaryFieldAccessor$DefaultFinalClassAccessor
> @ 0xef2e7568
> class org.apache.ignite.internal.binary.BinaryFieldAccessor @ 0xef304570
> class org.apache.ignite.internal.binary.BinaryClassDescriptor @ 0xef30d458
> class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
> class org.apache.ignite.internal.binary.BinaryReaderExImpl @ 0xef1a6708
> class org.apache.ignite.internal.binary.GridBinaryMarshaller @ 0xef300f88
> class org.apache.ignite.internal.binary.BinaryMarshaller @ 0xeefbdbc8
> class org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller @
> 0xeeee22b0
> class org.apache.ignite.internal.util.IgniteUtils @ 0xeed806d0
> class
> org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$7 @
> 0xef77ec78
> class org.apache.ignite.internal.managers.communication.GridIoManager @
> 0xef3304e0
> class org.apache.ignite.internal.managers.communication.GridIoManager @
> 0xef3304e0
> class org.apache.ignite.internal.managers.communication.GridIoManager @
> 0xef3304e0
> class org.apache.ignite.internal.managers.communication.GridIoManager$9 @
> 0xef6406e8
> class java.util.concurrent.ThreadPoolExecutor @ 0xeab0fec0
> class java.util.concurrent.ThreadPoolExecutor$Worker @ 0xeae473e8
> class java.lang.Thread @ 0xeaaf0360
> {code}
> !image-2021-12-15-21-13-43-775.png!
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
