Artem Plotnikov created ZEPPELIN-4623:
-----------------------------------------
Summary: Deadlock in Zeppelin during interpreter restart
Key: ZEPPELIN-4623
URL: https://issues.apache.org/jira/browse/ZEPPELIN-4623
Project: Zeppelin
Issue Type: Bug
Components: Core
Affects Versions: 0.8.2
Reporter: Artem Plotnikov
Sometimes interpreter restart results in server deadlock, which can be caught
with `jcmd <pid> Thread.print` with output like:
{code:java}
Found one Java-level deadlock:
=============================
"qtp846238611-295773":
waiting to lock monitor 0x00007f3cdc099408 (object 0x0000000673c66758, a
java.util.LinkedList),
which is held by "qtp846238611-295335"
"qtp846238611-295335":
waiting to lock monitor 0x00007f3e98023cd8 (object 0x000000065accda50, a
java.util.ArrayList),
which is held by "qtp846238611-292136"
"qtp846238611-292136":
waiting to lock monitor 0x00007f3cdc099408 (object 0x0000000673c66758, a
java.util.LinkedList),
which is held by "qtp846238611-295335"Java stack information for the threads
listed above:
===================================================
"qtp846238611-295773":
at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750)
- waiting to lock <0x0000000673c66758> (a java.util.LinkedList)
at
org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872)
at
org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568)
at
org.apache.zeppelin.socket.NotebookServer$ParagraphListenerImpl.afterStatusChange(NotebookServer.java:2333)
at org.apache.zeppelin.scheduler.Job.setStatus(Job.java:162)
at
org.apache.zeppelin.scheduler.RemoteScheduler.submit(RemoteScheduler.java:154)
at org.apache.zeppelin.notebook.Paragraph.execute(Paragraph.java:355)
at org.apache.zeppelin.notebook.Note.run(Note.java:683)
at
org.apache.zeppelin.socket.NotebookServer.persistAndExecuteSingleParagraph(NotebookServer.java:1881)
at
org.apache.zeppelin.socket.NotebookServer.runParagraph(NotebookServer.java:1840)
at
org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:262)
at
org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189)
at
org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162)
at
org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459)
at
org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105)
at
org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85)
at
org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220)
at
org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219)
at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
"qtp846238611-295335":
at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750)
- waiting to lock <0x000000065accda50> (a java.util.ArrayList)
at
org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872)
at
org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568)
at
org.apache.zeppelin.socket.NotebookServer$NotebookInformationListener.onParagraphRemove(NotebookServer.java:2194)
at
org.apache.zeppelin.notebook.Notebook.onParagraphRemove(Notebook.java:1084)
at org.apache.zeppelin.notebook.Note.removeParagraph(Note.java:446)
- locked <0x0000000673c66758> (a java.util.LinkedList)
at
org.apache.zeppelin.socket.NotebookServer.removeParagraph(NotebookServer.java:1363)
at
org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:283)
at
org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189)
at
org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162)
at
org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459)
at
org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105)
at
org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85)
at
org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220)
at
org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219)
at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
"qtp846238611-292136":
at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750)
- waiting to lock <0x0000000673c66758> (a java.util.LinkedList)
at
org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872)
at
org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568)
at
org.apache.zeppelin.socket.NotebookServer$NotebookInformationListener.onParagraphRemove(NotebookServer.java:2194)
at
org.apache.zeppelin.notebook.Notebook.onParagraphRemove(Notebook.java:1084)
at org.apache.zeppelin.notebook.Note.removeParagraph(Note.java:446)
- locked <0x000000065accda50> (a java.util.ArrayList)
at
org.apache.zeppelin.socket.NotebookServer.removeParagraph(NotebookServer.java:1363)
at
org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:283)
at
org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189)
at
org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66)
at
org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158)
at
org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162)
at
org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459)
at
org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105)
at
org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142)
at
org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85)
at
org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220)
at
org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219)
at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559)
at
org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
