Repository: metron Updated Branches: refs/heads/master ce381f3e0 -> fee758b3f
METRON-961: Spooky intermittent exceptions in pcap during sync should be handled and explained closes apache/incubator-metron#592 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fee758b3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fee758b3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fee758b3 Branch: refs/heads/master Commit: fee758b3fb2277067421a95054c3e328730decda Parents: ce381f3 Author: cstella <[email protected]> Authored: Sat May 20 09:55:42 2017 -0400 Committer: cstella <[email protected]> Committed: Sat May 20 09:55:42 2017 -0400 ---------------------------------------------------------------------- .../metron/spout/pcap/PartitionHDFSWriter.java | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/fee758b3/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java index 86697db..ebd60dc 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/spout/pcap/PartitionHDFSWriter.java @@ -65,11 +65,10 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable { }) ,HDFS(new SyncHandler() { @Override - public void sync(FSDataOutputStream outputStream) throws IOException{ - + public void sync(FSDataOutputStream outputStream) throws IOException { outputStream.hflush(); outputStream.hsync(); - ((HdfsDataOutputStream)outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); + ((HdfsDataOutputStream) outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } }) ,LOCAL(new SyncHandler() { @@ -91,8 +90,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable { } @Override - public void sync(FSDataOutputStream input) throws IOException { - func.sync(input); + public void sync(FSDataOutputStream input) { + try { + func.sync(input); + } + catch(IOException ioe) { + LOG.warn("Problems during sync, but this shouldn't be too concerning as long as it's intermittent: " + ioe.getMessage(), ioe); + } } } @@ -156,7 +160,13 @@ public class PartitionHDFSWriter implements AutoCloseable, Serializable { public void handle(long ts, byte[] value) throws IOException { turnoverIfNecessary(ts); BytesWritable bw = new BytesWritable(value); - writer.append(new LongWritable(ts), bw); + try { + writer.append(new LongWritable(ts), bw); + } + catch(ArrayIndexOutOfBoundsException aioobe) { + LOG.warn("This appears to be HDFS-7765 (https://issues.apache.org/jira/browse/HDFS-7765), " + + "which is an issue with syncing and not problematic: " + aioobe.getMessage(), aioobe); + } numWritten++; if(numWritten % config.getSyncEvery() == 0) { syncHandler.sync(outputStream);
