[
https://issues.apache.org/jira/browse/SSHD-539?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Goldstein Lyor resolved SSHD-539.
---------------------------------
Resolution: Duplicate
Duplicates SSHD-557
> MINA SSHD deadlocks in channel.Window.waitForSpace() under high load
> --------------------------------------------------------------------
>
> Key: SSHD-539
> URL: https://issues.apache.org/jira/browse/SSHD-539
> Project: MINA SSHD
> Issue Type: Bug
> Affects Versions: 0.14.0
> Reporter: Sam Gardner
> Labels: channel
>
> I have a small project which makes some simple modifications to the SSHD
> 0.14.0 codebase - mostly, it dynamically rewrites remotely-forwarded port
> requests so that multiple clients may request remote forwards to the same
> port without conflicting with one another.
> Sometimes, when we run a high load on the server (I can reliably reproduce
> with four clients running ~10Mb/s file transfers over locally forwarded
> ports), the SSHD server deadlocks.
> At this point, the Tomcat instance running the server is still running, but
> requests to the SSHD are not received. This means that SSH requests from a
> client to the server time out, and I can {{telnet}} to the bound port, but I
> do not see any SSH banner.
> If I run {{jstack}} on the deadlocked Tomcat PID I get traces like this:
> {noformat}
> "sshd-SshServer[7cc04f3]-nio2-thread-5" prio=5 tid=0x00007ffa4fd24800
> nid=0x3c0f in Object.wait() [0x0000000116b59000]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000708f6c590> (a
> org.apache.sshd.common.channel.Window)
> at java.lang.Object.wait(Object.java:503)
> at org.apache.sshd.common.channel.Window.waitForSpace(Window.java:152)
> - locked <0x0000000708f6c590> (a
> org.apache.sshd.common.channel.Window)
> at
> org.apache.sshd.common.channel.ChannelOutputStream.flush(ChannelOutputStream.java:116)
> - locked <0x0000000708f5c930> (a
> org.apache.sshd.common.channel.ChannelOutputStream)
> at
> org.apache.sshd.common.forward.TcpipServerChannel$1.messageReceived(TcpipServerChannel.java:121)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:184)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker$2.run(Invoker.java:218)
> at
> sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
> and
> {noformat}
> "sshd-SshServer[7cc04f3]-nio2-thread-4" prio=5 tid=0x00007ffa4e8ce800
> nid=0x5a07 in Object.wait() [0x0000000116a4c000]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000708fa5b40> (a
> org.apache.sshd.common.channel.Window)
> at java.lang.Object.wait(Object.java:503)
> at org.apache.sshd.common.channel.Window.waitForSpace(Window.java:152)
> - locked <0x0000000708fa5b40> (a
> org.apache.sshd.common.channel.Window)
> at
> org.apache.sshd.common.channel.ChannelOutputStream.flush(ChannelOutputStream.java:116)
> - locked <0x0000000708fa39b0> (a
> org.apache.sshd.common.channel.ChannelOutputStream)
> at
> org.apache.sshd.common.forward.TcpipServerChannel$1.messageReceived(TcpipServerChannel.java:121)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:184)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker.invokeDirect(Invoker.java:157)
> at
> sun.nio.ch.UnixAsynchronousSocketChannelImpl.implRead(UnixAsynchronousSocketChannelImpl.java:553)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:275)
> at
> sun.nio.ch.AsynchronousSocketChannelImpl.read(AsynchronousSocketChannelImpl.java:296)
> at
> java.nio.channels.AsynchronousSocketChannel.read(AsynchronousSocketChannel.java:407)
> at
> org.apache.sshd.common.io.nio2.Nio2Session.startReading(Nio2Session.java:170)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:186)
> at
> org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:1)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler$1.run(Nio2CompletionHandler.java:32)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:30)
> at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
> at sun.nio.ch.Invoker$2.run(Invoker.java:218)
> at
> sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}.
> Superficially, this looks very similar to SSHD-348.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
