[
https://issues.apache.org/jira/browse/DRILL-4708?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16250625#comment-16250625
]
Chun Chang commented on DRILL-4708:
-----------------------------------
We might have two issues here. One is client to server connection. This is the
test Lenin did. I downloaded and tested with Lenin's data on a distributed
cluster, it worked for me as well. I am thinking we had some changes in that
area and what Lenin experienced might have been fix (I tested with MapR
1.11.0). The other is communication between drillbits. This issue seems still
there. The following is from automation run against current apache master
(1.12.0). I am reopening the JIRA so we can investigate this issue.
{noformat}
/root/drillAutomation/framework-master/framework/resources/Functional/data-shapes/wide-columns/5000/1000rows/parquet/q123.q
Query:
select SUBSTRING(str_var, 50, 20000) from widestrings
Failed with exception
java.sql.SQLException: SYSTEM ERROR: ChannelClosedException: Channel closed
/10.10.104.85:60238 <--> atsqa6c86.qa.lab/10.10.104.86:31010.
Query submission to Drillbit failed.
[Error Id: 1df96d41-2dc1-49f5-8a3a-398ff0cb86d1 ]
at
org.apache.drill.jdbc.impl.DrillCursor.nextRowInternally(DrillCursor.java:489)
at
org.apache.drill.jdbc.impl.DrillCursor.loadInitialSchema(DrillCursor.java:561)
at
org.apache.drill.jdbc.impl.DrillResultSetImpl.execute(DrillResultSetImpl.java:1895)
at
org.apache.drill.jdbc.impl.DrillResultSetImpl.execute(DrillResultSetImpl.java:61)
at
oadd.org.apache.calcite.avatica.AvaticaConnection$1.execute(AvaticaConnection.java:473)
at
org.apache.drill.jdbc.impl.DrillMetaImpl.prepareAndExecute(DrillMetaImpl.java:1100)
at
oadd.org.apache.calcite.avatica.AvaticaConnection.prepareAndExecuteInternal(AvaticaConnection.java:477)
at
org.apache.drill.jdbc.impl.DrillConnectionImpl.prepareAndExecuteInternal(DrillConnectionImpl.java:181)
at
oadd.org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:110)
at
oadd.org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:130)
at
org.apache.drill.jdbc.impl.DrillStatementImpl.executeQuery(DrillStatementImpl.java:112)
at
org.apache.drill.test.framework.DrillTestJdbc.executeSetupQuery(DrillTestJdbc.java:195)
at
org.apache.drill.test.framework.DrillTestJdbc.run(DrillTestJdbc.java:97)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Caused by: oadd.org.apache.drill.common.exceptions.UserException: SYSTEM ERROR:
ChannelClosedException: Channel closed /10.10.104.85:60238 <-->
atsqa6c86.qa.lab/10.10.104.86:31010.
Query submission to Drillbit failed.
[Error Id: 1df96d41-2dc1-49f5-8a3a-398ff0cb86d1 ]
at
oadd.org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:586)
at
oadd.org.apache.drill.exec.rpc.user.QueryResultHandler$SubmissionListener.failed(QueryResultHandler.java:294)
at
oadd.org.apache.drill.exec.rpc.RequestIdMap$RpcListener.setException(RequestIdMap.java:139)
at
oadd.org.apache.drill.exec.rpc.RequestIdMap$SetExceptionProcedure.apply(RequestIdMap.java:76)
at
oadd.org.apache.drill.exec.rpc.RequestIdMap$SetExceptionProcedure.apply(RequestIdMap.java:66)
at
oadd.com.carrotsearch.hppc.IntObjectHashMap.forEach(IntObjectHashMap.java:692)
at
oadd.org.apache.drill.exec.rpc.RequestIdMap.channelClosed(RequestIdMap.java:62)
at
oadd.org.apache.drill.exec.rpc.AbstractRemoteConnection.channelClosed(AbstractRemoteConnection.java:192)
at
oadd.org.apache.drill.exec.rpc.AbstractClientConnection.channelClosed(AbstractClientConnection.java:97)
at
oadd.org.apache.drill.exec.rpc.RpcBus$ChannelClosedHandler.operationComplete(RpcBus.java:167)
at
oadd.org.apache.drill.exec.rpc.RpcBus$ChannelClosedHandler.operationComplete(RpcBus.java:144)
at
oadd.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
at
oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
at
oadd.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
at
oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
at
oadd.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)
at
oadd.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82)
at
oadd.io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1003)
at
oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:632)
at
oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:610)
at
oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:553)
at
oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:71)
at
oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:89)
at
oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:162)
at
oadd.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
at
oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
at
oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
at oadd.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
at
oadd.io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
... 1 more
Caused by: oadd.org.apache.drill.exec.rpc.ChannelClosedException: Channel
closed /10.10.104.85:60238 <--> atsqa6c86.qa.lab/10.10.104.86:31010.
at
oadd.org.apache.drill.exec.rpc.RpcBus$ChannelClosedHandler.operationComplete(RpcBus.java:164)
... 20 more
{noformat}
> connection closed unexpectedly
> ------------------------------
>
> Key: DRILL-4708
> URL: https://issues.apache.org/jira/browse/DRILL-4708
> Project: Apache Drill
> Issue Type: Bug
> Components: Execution - RPC
> Affects Versions: 1.7.0
> Reporter: Chun Chang
> Assignee: Karthikeyan Manivannan
> Priority: Critical
> Attachments: data.tgz
>
>
> Running DRILL functional automation, we often see query failed randomly due
> to the following unexpected connection close error.
> {noformat}
> Execution Failures:
> /root/drillAutomation/framework/framework/resources/Functional/ctas/ctas_flatten/100000rows/filter5.q
> Query:
> select * from dfs.ctas_flatten.`filter5_100000rows_ctas`
> Failed with exception
> java.sql.SQLException: CONNECTION ERROR: Connection /10.10.100.171:36185 <-->
> drillats4.qa.lab/10.10.100.174:31010 (user client) closed unexpectedly.
> Drillbit down?
> [Error Id: 3d5dad8e-80d0-4c7f-9012-013bf01ce2b7 ]
> at
> org.apache.drill.jdbc.impl.DrillCursor.nextRowInternally(DrillCursor.java:247)
> at org.apache.drill.jdbc.impl.DrillCursor.next(DrillCursor.java:321)
> at
> oadd.net.hydromatic.avatica.AvaticaResultSet.next(AvaticaResultSet.java:187)
> at
> org.apache.drill.jdbc.impl.DrillResultSetImpl.next(DrillResultSetImpl.java:172)
> at
> org.apache.drill.test.framework.DrillTestJdbc.executeQuery(DrillTestJdbc.java:210)
> at
> org.apache.drill.test.framework.DrillTestJdbc.run(DrillTestJdbc.java:99)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:744)
> Caused by: oadd.org.apache.drill.common.exceptions.UserException: CONNECTION
> ERROR: Connection /10.10.100.171:36185 <-->
> drillats4.qa.lab/10.10.100.174:31010 (user client) closed unexpectedly.
> Drillbit down?
> [Error Id: 3d5dad8e-80d0-4c7f-9012-013bf01ce2b7 ]
> at
> oadd.org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:543)
> at
> oadd.org.apache.drill.exec.rpc.user.QueryResultHandler$ChannelClosedHandler$1.operationComplete(QueryResultHandler.java:373)
> at
> oadd.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:680)
> at
> oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:603)
> at
> oadd.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:563)
> at
> oadd.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:406)
> at
> oadd.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82)
> at
> oadd.io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:943)
> at
> oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:592)
> at
> oadd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:584)
> at
> oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:71)
> at
> oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.handleReadException(AbstractNioByteChannel.java:89)
> at
> oadd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:162)
> at
> oadd.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:511)
> at
> oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468)
> at
> oadd.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382)
> at oadd.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354)
> at
> oadd.io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> ... 1 more
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
