[
https://issues.apache.org/jira/browse/ASTERIXDB-2382?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Till reassigned ASTERIXDB-2382:
-------------------------------
Assignee: Michael Blow
> An NC instance becomes FAILED status after generating "TCP read error"
> ----------------------------------------------------------------------
>
> Key: ASTERIXDB-2382
> URL: https://issues.apache.org/jira/browse/ASTERIXDB-2382
> Project: Apache AsterixDB
> Issue Type: Bug
> Affects Versions: 0.9.4
> Reporter: Taewoo Kim
> Assignee: Michael Blow
> Priority: Major
> Fix For: 0.9.4.2
>
>
> An NC belongs to the Cloudberry cluster becomes FAILED status after
> generating the following exceptions.
>
> {code:java}
> 00:08:07.406 [TCPEndpoint IO Thread] ERROR
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
> java.io.IOException: Invalid argument
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
> [hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> 00:08:07.464 [Executor-209108:4] WARN org.apache.hyracks.control.nc.Task -
> Task TAID:TID:ANID:ODID:1:1:6:0 failed with exception
> org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> [hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
> Suppressed: org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:149)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:93)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> [hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.error(AbstractChannelWriteInterface.java:146)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.fail(NetworkOutputChannel.java:97)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.fail(PipelinedPartition.java:105)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:147)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 9 more
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
> ~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 8 more
> 00:08:07.484 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue
> - Executing: NotifyTaskFailure
> 00:08:07.464 [Executor-209090:4] WARN org.apache.hyracks.control.nc.Task -
> Task TAID:TID:ANID:ODID:1:1:7:0 failed with exception
> org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> [hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
> Suppressed: org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:149)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:93)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> [hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.error(AbstractChannelWriteInterface.java:146)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.fail(NetworkOutputChannel.java:97)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.fail(PipelinedPartition.java:105)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.fail(PartitionDataWriter.java:147)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 9 more
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
> ~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 8 more
> 00:08:07.484 [Worker:4] WARN
> org.apache.hyracks.control.nc.work.NotifyTaskFailureWork - 4 is sending a
> notification to cc that task TAID:TID:ANID:ODID:1:1:6:0 has failed
> org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> ~[?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> ~[?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) ~[?:1.8.0]
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
> ~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 8 more
> 00:08:07.484 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue
> - Executing: NotifyTaskFailure
> 00:08:07.484 [Worker:4] WARN
> org.apache.hyracks.control.nc.work.NotifyTaskFailureWork - 4 is sending a
> notification to cc that task TAID:TID:ANID:ODID:1:1:7:0 has failed
> org.apache.hyracks.api.exceptions.HyracksDataException:
> java.nio.channels.CancelledKeyException
> at
> org.apache.hyracks.api.exceptions.HyracksDataException.create(HyracksDataException.java:51)
> ~[hyracks-api-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.wrapException(PartitionDataWriter.java:178)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:87)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime.close(AbstractOneInputOneOutputOneFramePushRuntime.java:62)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor$1.close(AlgebricksMetaOperatorDescriptor.java:156)
> ~[algebricks-runtime-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.close(OptimizedHybridHashJoinOperatorDescriptor.java:465)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:410)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at org.apache.hyracks.control.nc.Task.run(Task.java:324)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> ~[?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> ~[?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) ~[?:1.8.0]
> Caused by: java.nio.channels.CancelledKeyException
> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
> ~[?:1.8.0]
> at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
> ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.tcp.TCPConnection.enable(TCPConnection.java:71)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection$1.increment(MultiplexedConnection.java:77)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelSet.markPendingWrite(ChannelSet.java:177)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.markPendingWrite(ChannelControlBlock.java:157)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface.adjustChannelWritability(AbstractChannelWriteInterface.java:89)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.AbstractChannelWriteInterface$CloseableBufferAcceptor.accept(AbstractChannelWriteInterface.java:124)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.comm.channels.NetworkOutputChannel.nextFrame(NetworkOutputChannel.java:91)
> ~[hyracks-comm-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.control.nc.partitions.PipelinedPartition.nextFrame(PipelinedPartition.java:82)
> ~[hyracks-control-nc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.common.comm.io.AbstractFrameAppender.write(AbstractFrameAppender.java:93)
> ~[hyracks-dataflow-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.dataflow.std.connectors.PartitionDataWriter.close(PartitionDataWriter.java:83)
> ~[hyracks-dataflow-std-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> ... 8 more
> 00:08:08.430 [IPC Network Listener Thread [/128.195.52.80:46417]] ERROR
> org.apache.hyracks.ipc.impl.IPCConnectionManager - TCP read error from
> /128.195.52.77:1099
> java.io.IOException: Invalid argument
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
> at
> org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.read(IPCConnectionManager.java:358)
> [hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:276)
> [hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196)
> [hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> 00:08:08.942 [TCPEndpoint IO Thread] ERROR
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
> java.io.IOException: Invalid argument
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
> [hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> 00:08:08.943 [TCPEndpoint IO Thread] ERROR
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint - Unexpected tcp io error
> java.io.IOException: Invalid argument
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0]
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0]
> at sun.nio.ch.IOUtil.read(IOUtil.java:197) ~[?:1.8.0]
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:375) ~[?:1.8.0]
> at
> org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelReadInterface.read(FullFrameChannelReadInterface.java:72)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock.read(ChannelControlBlock.java:90)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.driveReaderStateMachine(MultiplexedConnection.java:418)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection.notifyIOReady(MultiplexedConnection.java:132)
> ~[hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.net.protocols.tcp.TCPEndpoint$IOThread.run(TCPEndpoint.java:179)
> [hyracks-net-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> 00:08:08.944 [Worker:4] INFO org.apache.hyracks.control.common.work.WorkQueue
> - Executing: NotifyTaskCompleteWork:TAID:TID:ANID:ODID:3:0:7:0
> 00:08:08.945 [Worker:4] WARN
> org.apache.hyracks.ipc.impl.ReconnectingIPCHandle - ipcHandle IPCHandle
> [addr=/128.195.52.77:1099 state=CLOSED] disconnected; will attempt to
> reconnect 1 times
> 00:08:08.994 [IPC Network Listener Thread [/128.195.52.80:46417]] ERROR
> org.apache.hyracks.ipc.impl.IPCConnectionManager - Exception processing
> message; sleeping 1 seconds
> java.net.SocketException: Network is unreachable
> at sun.nio.ch.Net.connect0(Native Method) ~[?:1.8.0]
> at sun.nio.ch.Net.connect(Net.java:435) ~[?:1.8.0]
> at sun.nio.ch.Net.connect(Net.java:427) ~[?:1.8.0]
> at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:643) ~[?:1.8.0]
> at
> org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.doRun(IPCConnectionManager.java:223)
> [hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.ipc.impl.IPCConnectionManager$NetworkThread.run(IPCConnectionManager.java:196)
> [hyracks-ipc-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> plutonium.ics.uci.edu
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
