Taewoo Kim created ASTERIXDB-2382:
-------------------------------------
Summary: An NC instance becomes FAILED status after generating
"TCP read error"
Key: ASTERIXDB-2382
URL: https://issues.apache.org/jira/browse/ASTERIXDB-2382
Project: Apache AsterixDB
Issue Type: Bug
Reporter: Taewoo Kim
An NC belongs to the Cloudberry cluster becomes FAILED status after generating
the following exceptions.
{code:java}
00:08:07.406 [TCPEndpoint IO Thread] ERROR
org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
java.io.IOException: Invalid argument
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
00:08:07.464 [Executor-209108:4] WARN org.apache.hyracks.control.nc.Task - Task
TAID:TID:ANID:ODID:1:1:6:0 failed with exception
org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
Suppressed: org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:149)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:93)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.error(AbstractChannelWriteInterface.java:146)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.fail(NetworkOutputChannel.java:97)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.fail(PipelinedPartition.java:105)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:147)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 9 more
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 8 more
00:08:07.484 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue -
Executing: NotifyTaskFailure
00:08:07.464 [Executor-209090:4] WARN org.apache.hyracks.control.nc.Task - Task
TAID:TID:ANID:ODID:1:1:7:0 failed with exception
org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
Suppressed: org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:149)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:93)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.error(AbstractChannelWriteInterface.java:146)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.fail(NetworkOutputChannel.java:97)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.fail(PipelinedPartition.java:105)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:147)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 9 more
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 8 more
00:08:07.484 [Worker:4] WARN
org.apache.hyracks.control.nc.work.NotifyTaskFailureWork - 4 is sending a
notification to cc that task TAID:TID:ANID:ODID:1:1:6:0 has failed
org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) ~[?:1.8.0]
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 8 more
00:08:07.484 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue -
Executing: NotifyTaskFailure
00:08:07.484 [Worker:4] WARN
org.apache.hyracks.control.nc.work.NotifyTaskFailureWork - 4 is sending a
notification to cc that task TAID:TID:ANID:ODID:1:1:7:0 has failed
org.apache.hyracks.api.exceptions.HyracksDataException:
java.nio.channels.CancelledKeyException
at
org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at org.apache.hyracks.control.nc.Task.run(Task.java:324)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) ~[?:1.8.0]
Caused by: java.nio.channels.CancelledKeyException
at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73) ~[?:1.8.0]
at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
... 8 more
00:08:08.430 [IPC Network Listener Thread [/128.195.52.80:46417]] ERROR
org.apache.hyracks.ipc.impl.IPCConnectionManager - TCP read error from
/128.195.52.77:1099
java.io.IOException: Invalid argument
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
at
org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.read(IPCConnectionManager.java:358)
[hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:276)
[hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196)
[hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
00:08:08.942 [TCPEndpoint IO Thread] ERROR
org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
java.io.IOException: Invalid argument
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
00:08:08.943 [TCPEndpoint IO Thread] ERROR
org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
java.io.IOException: Invalid argument
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
at
org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
00:08:08.944 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue -
Executing: NotifyTaskCompleteWork:TAID:TID:ANID:ODID:3:0:7:0
00:08:08.945 [Worker:4] WARN org.apache.hyracks.ipc.impl.ReconnectingIPCHandle
- ipcHandle IPCHandle [addr=/128.195.52.77:1099 state=CLOSED] disconnected;
will attempt to reconnect 1 times
00:08:08.994 [IPC Network Listener Thread [/128.195.52.80:46417]] ERROR
org.apache.hyracks.ipc.impl.IPCConnectionManager - Exception processing
message; sleeping 1 seconds
java.net.SocketException: Network is unreachable
at sun.nio.ch.Net.connect0(Native Method) ~[?:1.8.0]
at sun.nio.ch.Net.connect(Net.java:435) ~[?:1.8.0]
at sun.nio.ch.Net.connect(Net.java:427) ~[?:1.8.0]
at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:643) ~[?:1.8.0]
at
org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:223)
[hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196)
[hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
plutonium.ics.uci.edu
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
