HADOOP-13895. Make FileStatus Serializable
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59c5f187 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59c5f187 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59c5f187 Branch: refs/heads/YARN-5734 Commit: 59c5f18784121f04030d1d0982f2e2285688ee11 Parents: 3619ae3 Author: Chris Douglas <[email protected]> Authored: Wed Feb 1 10:19:36 2017 -0800 Committer: Chris Douglas <[email protected]> Committed: Wed Feb 1 10:19:36 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/BlockLocation.java | 7 +++++-- .../java/org/apache/hadoop/fs/FileStatus.java | 21 +++++++++++++++++-- .../hadoop/fs/permission/FsCreateModes.java | 1 + .../hadoop/fs/permission/FsPermission.java | 22 +++++++++++++++++--- .../org/apache/hadoop/fs/TestFileStatus.java | 19 +++++++++++++++++ .../org/apache/hadoop/fs/HdfsBlockLocation.java | 17 +++++++++++++-- .../hdfs/protocol/FsPermissionExtension.java | 2 ++ 7 files changed, 80 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java index 7811ef5..b8cad3a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; import java.io.IOException; +import java.io.Serializable; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -29,7 +30,9 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class BlockLocation { +public class BlockLocation implements Serializable { + private static final long serialVersionUID = 0x22986f6d; + private String[] hosts; // Datanode hostnames private String[] cachedHosts; // Datanode hostnames with a cached replica private String[] names; // Datanode IP:xferPort for accessing the block @@ -303,4 +306,4 @@ public class BlockLocation { } return result.toString(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 6a79768..72ca24f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -20,6 +20,9 @@ package org.apache.hadoop.fs; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.ObjectInputValidation; +import java.io.Serializable; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -31,11 +34,14 @@ import org.apache.hadoop.io.Writable; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class FileStatus implements Writable, Comparable<FileStatus> { +public class FileStatus implements Writable, Comparable<FileStatus>, + Serializable, ObjectInputValidation { + + private static final long serialVersionUID = 0x13caeae8; private Path path; private long length; - private boolean isdir; + private Boolean isdir; private short block_replication; private long blocksize; private long modification_time; @@ -387,4 +393,15 @@ public class FileStatus implements Writable, Comparable<FileStatus> { sb.append("}"); return sb.toString(); } + + @Override + public void validateObject() throws InvalidObjectException { + if (null == path) { + throw new InvalidObjectException("No Path in deserialized FileStatus"); + } + if (null == isdir) { + throw new InvalidObjectException("No type in deserialized FileStatus"); + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java index a1ed0d7..2bd6f1f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Evolving public final class FsCreateModes extends FsPermission { + private static final long serialVersionUID = 0x22986f6d; private final FsPermission unmasked; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java index fabfc12..56e19dc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java @@ -20,6 +20,9 @@ package org.apache.hadoop.fs.permission; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.ObjectInputValidation; +import java.io.Serializable; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,8 +39,10 @@ import org.apache.hadoop.io.WritableFactory; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class FsPermission implements Writable { +public class FsPermission implements Writable, Serializable, + ObjectInputValidation { private static final Log LOG = LogFactory.getLog(FsPermission.class); + private static final long serialVersionUID = 0x2fe08564; static final WritableFactory FACTORY = new WritableFactory() { @Override @@ -60,7 +65,7 @@ public class FsPermission implements Writable { private FsAction useraction = null; private FsAction groupaction = null; private FsAction otheraction = null; - private boolean stickyBit = false; + private Boolean stickyBit = false; private FsPermission() {} @@ -202,7 +207,7 @@ public class FsPermission implements Writable { return this.useraction == that.useraction && this.groupaction == that.groupaction && this.otheraction == that.otheraction - && this.stickyBit == that.stickyBit; + && this.stickyBit.booleanValue() == that.stickyBit.booleanValue(); } return false; } @@ -377,6 +382,7 @@ public class FsPermission implements Writable { } private static class ImmutableFsPermission extends FsPermission { + private static final long serialVersionUID = 0x1bab54bd; public ImmutableFsPermission(short permission) { super(permission); } @@ -386,4 +392,14 @@ public class FsPermission implements Writable { throw new UnsupportedOperationException(); } } + + @Override + public void validateObject() throws InvalidObjectException { + if (null == useraction || null == groupaction || null == otheraction) { + throw new InvalidObjectException("Invalid mode in FsPermission"); + } + if (null == stickyBit) { + throw new InvalidObjectException("No sticky bit in FsPermission"); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java index dd5279d..35f2bad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java @@ -26,6 +26,8 @@ import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -216,6 +218,23 @@ public class TestFileStatus { MTIME, ATIME, PERMISSION, OWNER, GROUP, symlink, PATH); validateToString(fileStatus); } + + @Test + public void testSerializable() throws Exception { + Path p = new Path("uqsf://ybpnyubfg:8020/sbb/one/onm"); + FsPermission perm = FsPermission.getFileDefault(); + FileStatus stat = new FileStatus(4344L, false, 4, 512L << 20, 12345678L, + 87654321L, perm, "yak", "dingo", p); + ByteArrayOutputStream baos = new ByteArrayOutputStream(1024); + try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(stat); + } + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + try (ObjectInputStream ois = new ObjectInputStream(bais)) { + FileStatus deser = (FileStatus) ois.readObject(); + assertEquals(stat, deser); + } + } /** * Validate the accessors for FileStatus. http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java index eac3f96..2ee7f41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.fs; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -28,9 +32,10 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class HdfsBlockLocation extends BlockLocation { +public class HdfsBlockLocation extends BlockLocation implements Serializable { + private static final long serialVersionUID = 0x7aecec92; - private final LocatedBlock block; + private transient LocatedBlock block; public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) { // Initialize with data from passed in BlockLocation @@ -41,4 +46,12 @@ public class HdfsBlockLocation extends BlockLocation { public LocatedBlock getLocatedBlock() { return block; } + + private void readObject(ObjectInputStream ois) + throws IOException, ClassNotFoundException { + ois.defaultReadObject(); + // LocatedBlock is not Serializable + block = null; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java index f74472d..786bb58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.permission.FsPermission; */ @InterfaceAudience.Private public class FsPermissionExtension extends FsPermission { + private static final long serialVersionUID = 0x13c298a4; + private final static short ACL_BIT = 1 << 12; private final static short ENCRYPTED_BIT = 1 << 13; private final boolean aclBit; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
