HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e13fc62 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e13fc62 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e13fc62 Branch: refs/heads/YARN-2139 Commit: 6e13fc62e1f284f22fd0089f06ce281198bc7c2a Parents: fae3e86 Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Mon Dec 15 10:00:25 2014 -0800 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Mon Dec 15 10:30:30 2014 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/inotify/Event.java | 87 ++++++++++++++++++-- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 19 +++-- .../namenode/InotifyFSEditLogOpTranslator.java | 34 ++++++-- .../hadoop-hdfs/src/main/proto/inotify.proto | 1 + .../hdfs/TestDFSInotifyEventInputStream.java | 4 +- 6 files changed, 124 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9dfecc1..99b06be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -457,6 +457,8 @@ Release 2.7.0 - UNRELEASED HDFS-7426. Change nntop JMX format to be a JSON blob. (wang) + HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java index e8a34e7..5ceff1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java @@ -101,6 +101,7 @@ public abstract class Event { private FsPermission perms; private String symlinkTarget; private boolean overwrite; + private long defaultBlockSize; public static class Builder { private INodeType iNodeType; @@ -112,6 +113,7 @@ public abstract class Event { private FsPermission perms; private String symlinkTarget; private boolean overwrite; + private long defaultBlockSize = 0; public Builder iNodeType(INodeType type) { this.iNodeType = type; @@ -158,6 +160,11 @@ public abstract class Event { return this; } + public Builder defaultBlockSize(long defaultBlockSize) { + this.defaultBlockSize = defaultBlockSize; + return this; + } + public CreateEvent build() { return new CreateEvent(this); } @@ -174,6 +181,7 @@ public abstract class Event { this.perms = b.perms; this.symlinkTarget = b.symlinkTarget; this.overwrite = b.overwrite; + this.defaultBlockSize = b.defaultBlockSize; } public INodeType getiNodeType() { @@ -220,6 +228,10 @@ public abstract class Event { public boolean getOverwrite() { return overwrite; } + + public long getDefaultBlockSize() { + return defaultBlockSize; + } } /** @@ -398,11 +410,36 @@ public abstract class Event { private String dstPath; private long timestamp; - public RenameEvent(String srcPath, String dstPath, long timestamp) { + public static class Builder { + private String srcPath; + private String dstPath; + private long timestamp; + + public Builder srcPath(String srcPath) { + this.srcPath = srcPath; + return this; + } + + public Builder dstPath(String dstPath) { + this.dstPath = dstPath; + return this; + } + + public Builder timestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + + public RenameEvent build() { + return new RenameEvent(this); + } + } + + private RenameEvent(Builder builder) { super(EventType.RENAME); - this.srcPath = srcPath; - this.dstPath = dstPath; - this.timestamp = timestamp; + this.srcPath = builder.srcPath; + this.dstPath = builder.dstPath; + this.timestamp = builder.timestamp; } public String getSrcPath() { @@ -427,9 +464,22 @@ public abstract class Event { public static class AppendEvent extends Event { private String path; - public AppendEvent(String path) { + public static class Builder { + private String path; + + public Builder path(String path) { + this.path = path; + return this; + } + + public AppendEvent build() { + return new AppendEvent(this); + } + } + + private AppendEvent(Builder b) { super(EventType.APPEND); - this.path = path; + this.path = b.path; } public String getPath() { @@ -444,10 +494,29 @@ public abstract class Event { private String path; private long timestamp; - public UnlinkEvent(String path, long timestamp) { + public static class Builder { + private String path; + private long timestamp; + + public Builder path(String path) { + this.path = path; + return this; + } + + public Builder timestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + + public UnlinkEvent build() { + return new UnlinkEvent(this); + } + } + + private UnlinkEvent(Builder builder) { super(EventType.UNLINK); - this.path = path; - this.timestamp = timestamp; + this.path = builder.path; + this.timestamp = builder.timestamp; } public String getPath() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 5a36585..ee6d58c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -2566,6 +2566,7 @@ public class PBHelper { .replication(create.getReplication()) .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null : create.getSymlinkTarget()) + .defaultBlockSize(create.getDefaultBlockSize()) .overwrite(create.getOverwrite()).build()); break; case EVENT_METADATA: @@ -2592,19 +2593,26 @@ public class PBHelper { case EVENT_RENAME: InotifyProtos.RenameEventProto rename = InotifyProtos.RenameEventProto.parseFrom(p.getContents()); - events.add(new Event.RenameEvent(rename.getSrcPath(), - rename.getDestPath(), rename.getTimestamp())); + events.add(new Event.RenameEvent.Builder() + .srcPath(rename.getSrcPath()) + .dstPath(rename.getDestPath()) + .timestamp(rename.getTimestamp()) + .build()); break; case EVENT_APPEND: InotifyProtos.AppendEventProto reopen = InotifyProtos.AppendEventProto.parseFrom(p.getContents()); - events.add(new Event.AppendEvent(reopen.getPath())); + events.add(new Event.AppendEvent.Builder() + .path(reopen.getPath()) + .build()); break; case EVENT_UNLINK: InotifyProtos.UnlinkEventProto unlink = InotifyProtos.UnlinkEventProto.parseFrom(p.getContents()); - events.add(new Event.UnlinkEvent(unlink.getPath(), - unlink.getTimestamp())); + events.add(new Event.UnlinkEvent.Builder() + .path(unlink.getPath()) + .timestamp(unlink.getTimestamp()) + .build()); break; default: throw new RuntimeException("Unexpected inotify event type: " + @@ -2650,6 +2658,7 @@ public class PBHelper { .setReplication(ce2.getReplication()) .setSymlinkTarget(ce2.getSymlinkTarget() == null ? "" : ce2.getSymlinkTarget()) + .setDefaultBlockSize(ce2.getDefaultBlockSize()) .setOverwrite(ce2.getOverwrite()).build().toByteString() ).build()); break; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java index cd3fc23..f265340 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java @@ -53,10 +53,13 @@ public class InotifyFSEditLogOpTranslator { .groupName(addOp.permissions.getGroupName()) .perms(addOp.permissions.getPermission()) .overwrite(addOp.overwrite) + .defaultBlockSize(addOp.blockSize) .iNodeType(Event.CreateEvent.INodeType.FILE).build() }); - } else { + } else { // append return new EventBatch(op.txid, - new Event[] { new Event.AppendEvent(addOp.path) }); + new Event[]{new Event.AppendEvent.Builder() + .path(addOp.path) + .build()}); } case OP_CLOSE: FSEditLogOp.CloseOp cOp = (FSEditLogOp.CloseOp) op; @@ -72,25 +75,40 @@ public class InotifyFSEditLogOpTranslator { case OP_CONCAT_DELETE: FSEditLogOp.ConcatDeleteOp cdOp = (FSEditLogOp.ConcatDeleteOp) op; List<Event> events = Lists.newArrayList(); - events.add(new Event.AppendEvent(cdOp.trg)); + events.add(new Event.AppendEvent.Builder() + .path(cdOp.trg) + .build()); for (String src : cdOp.srcs) { - events.add(new Event.UnlinkEvent(src, cdOp.timestamp)); + events.add(new Event.UnlinkEvent.Builder() + .path(src) + .timestamp(cdOp.timestamp) + .build()); } events.add(new Event.CloseEvent(cdOp.trg, -1, cdOp.timestamp)); return new EventBatch(op.txid, events.toArray(new Event[0])); case OP_RENAME_OLD: FSEditLogOp.RenameOldOp rnOpOld = (FSEditLogOp.RenameOldOp) op; return new EventBatch(op.txid, new Event[] { - new Event.RenameEvent(rnOpOld.src, - rnOpOld.dst, rnOpOld.timestamp) }); + new Event.RenameEvent.Builder() + .srcPath(rnOpOld.src) + .dstPath(rnOpOld.dst) + .timestamp(rnOpOld.timestamp) + .build() }); case OP_RENAME: FSEditLogOp.RenameOp rnOp = (FSEditLogOp.RenameOp) op; return new EventBatch(op.txid, new Event[] { - new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) }); + new Event.RenameEvent.Builder() + .srcPath(rnOp.src) + .dstPath(rnOp.dst) + .timestamp(rnOp.timestamp) + .build() }); case OP_DELETE: FSEditLogOp.DeleteOp delOp = (FSEditLogOp.DeleteOp) op; return new EventBatch(op.txid, new Event[] { - new Event.UnlinkEvent(delOp.path, delOp.timestamp) }); + new Event.UnlinkEvent.Builder() + .path(delOp.path) + .timestamp(delOp.timestamp) + .build() }); case OP_MKDIR: FSEditLogOp.MkdirOp mkOp = (FSEditLogOp.MkdirOp) op; return new EventBatch(op.txid, http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto index e51c02c..e50f14b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto @@ -78,6 +78,7 @@ message CreateEventProto { optional int32 replication = 7; optional string symlinkTarget = 8; optional bool overwrite = 9; + optional int64 defaultBlockSize = 10 [default=0]; } message CloseEventProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e13fc62/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java index 82db110..77a17ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java @@ -170,6 +170,7 @@ public class TestDFSInotifyEventInputStream { Assert.assertTrue(ce.getReplication() > 0); Assert.assertTrue(ce.getSymlinkTarget() == null); Assert.assertTrue(ce.getOverwrite()); + Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize()); // CloseOp batch = waitForNextEvents(eis); @@ -186,7 +187,8 @@ public class TestDFSInotifyEventInputStream { Assert.assertEquals(1, batch.getEvents().length); txid = checkTxid(batch, txid); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND); - Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2")); + Event.AppendEvent append2 = (Event.AppendEvent)batch.getEvents()[0]; + Assert.assertEquals("/file2", append2.getPath()); // CloseOp batch = waitForNextEvents(eis);