HDFS-12681. Make HdfsLocatedFileStatus a subtype of LocatedFileStatus
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e560f3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e560f3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e560f3b Branch: refs/heads/HDFS-7240 Commit: 0e560f3b8d194c10dce06443979df4074e14b0db Parents: c9a54aa Author: Chris Douglas <[email protected]> Authored: Wed Nov 29 20:28:06 2017 -0800 Committer: Chris Douglas <[email protected]> Committed: Wed Nov 29 20:28:06 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileStatus.java | 61 ++- .../org/apache/hadoop/fs/LocatedFileStatus.java | 42 +- .../apache/hadoop/fs/protocolPB/PBHelper.java | 11 +- .../dev-support/findbugsExcludeFile.xml | 30 ++ .../hadoop/hdfs/protocol/HdfsFileStatus.java | 530 ++++++++++--------- .../hdfs/protocol/HdfsLocatedFileStatus.java | 208 ++++++-- .../hdfs/protocol/HdfsNamedFileStatus.java | 180 +++++++ .../hadoop/hdfs/protocol/package-info.java | 18 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 57 +- .../protocol/TestHdfsFileStatusMethods.java | 106 ++++ .../hadoop/fs/http/client/HttpFSFileSystem.java | 5 +- .../dev-support/findbugsExcludeFile.xml | 7 - .../apache/hadoop/hdfs/server/mover/Mover.java | 2 +- .../server/namenode/FSDirStatAndListingOp.java | 47 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +- .../hdfs/TestFileStatusSerialization.java | 8 +- .../hdfs/server/mover/TestStorageMover.java | 2 +- 17 files changed, 925 insertions(+), 391 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index d7c05d7..0663c43 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -56,15 +56,36 @@ public class FileStatus implements Writable, Comparable<Object>, private Path symlink; private Set<AttrFlags> attr; - private enum AttrFlags { + /** + * Flags for entity attributes. + */ + public enum AttrFlags { + /** ACL information available for this entity. */ HAS_ACL, + /** Entity is encrypted. */ HAS_CRYPT, + /** Entity is stored erasure-coded. */ HAS_EC, - SNAPSHOT_ENABLED + /** Snapshot capability enabled. */ + SNAPSHOT_ENABLED, } - private static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet(); - private static Set<AttrFlags> flags(boolean acl, boolean crypt, boolean ec) { - if (!(acl || crypt || ec)) { + + /** + * Shared, empty set of attributes (a common case for FileStatus). + */ + public static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet(); + + /** + * Convert boolean attributes to a set of flags. + * @param acl See {@link AttrFlags#HAS_ACL}. + * @param crypt See {@link AttrFlags#HAS_CRYPT}. + * @param ec See {@link AttrFlags#HAS_EC}. + * @param sn See {@link AttrFlags#SNAPSHOT_ENABLED}. + * @return converted set of flags. + */ + public static Set<AttrFlags> attributes(boolean acl, boolean crypt, + boolean ec, boolean sn) { + if (!(acl || crypt || ec || sn)) { return NONE; } EnumSet<AttrFlags> ret = EnumSet.noneOf(AttrFlags.class); @@ -77,6 +98,9 @@ public class FileStatus implements Writable, Comparable<Object>, if (ec) { ret.add(AttrFlags.HAS_EC); } + if (sn) { + ret.add(AttrFlags.SNAPSHOT_ENABLED); + } return ret; } @@ -117,6 +141,15 @@ public class FileStatus implements Writable, Comparable<Object>, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, Path symlink, Path path, boolean hasAcl, boolean isEncrypted, boolean isErasureCoded) { + this(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, + attributes(hasAcl, isEncrypted, isErasureCoded, false)); + } + + public FileStatus(long length, boolean isdir, int block_replication, + long blocksize, long modification_time, long access_time, + FsPermission permission, String owner, String group, Path symlink, + Path path, Set<AttrFlags> attr) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -136,7 +169,7 @@ public class FileStatus implements Writable, Comparable<Object>, this.group = (group == null) ? "" : group; this.symlink = symlink; this.path = path; - attr = flags(hasAcl, isEncrypted, isErasureCoded); + this.attr = attr; // The variables isdir and symlink indicate the type: // 1. isdir implies directory, in which case symlink must be null. @@ -341,19 +374,6 @@ public class FileStatus implements Writable, Comparable<Object>, } /** - * Sets Snapshot enabled flag. - * - * @param isSnapShotEnabled When true, SNAPSHOT_ENABLED flag is set - */ - public void setSnapShotEnabledFlag(boolean isSnapShotEnabled) { - if (isSnapShotEnabled) { - attr.add(AttrFlags.SNAPSHOT_ENABLED); - } else { - attr.remove(AttrFlags.SNAPSHOT_ENABLED); - } - } - - /** * @return The contents of the symbolic link. */ public Path getSymlink() throws IOException { @@ -480,7 +500,8 @@ public class FileStatus implements Writable, Comparable<Object>, setGroup(other.getGroup()); setSymlink((other.isSymlink() ? other.getSymlink() : null)); setPath(other.getPath()); - attr = flags(other.hasAcl(), other.isEncrypted(), other.isErasureCoded()); + attr = attributes(other.hasAcl(), other.isEncrypted(), + other.isErasureCoded(), other.isSnapshotEnabled()); assert (isDirectory() && getSymlink() == null) || !isDirectory(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java index 29e1998..4994957 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs; import java.io.IOException; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -115,9 +116,35 @@ public class LocatedFileStatus extends FileStatus { Path symlink, Path path, boolean hasAcl, boolean isEncrypted, boolean isErasureCoded, BlockLocation[] locations) { - super(length, isdir, block_replication, blocksize, modification_time, + this(length, isdir, block_replication, blocksize, modification_time, access_time, permission, owner, group, symlink, path, - hasAcl, isEncrypted, isErasureCoded); + attributes(hasAcl, isEncrypted, isErasureCoded, false), locations); + this.locations = locations; + } + + /** + * Constructor. + * + * @param length a file's length + * @param isdir if the path is a directory + * @param block_replication the file's replication factor + * @param blocksize a file's block size + * @param modification_time a file's modification time + * @param access_time a file's access time + * @param permission a file's permission + * @param owner a file's owner + * @param group a file's group + * @param symlink symlink if the path is a symbolic link + * @param path the path's qualified name + * @param attr Attribute flags (See {@link FileStatus.AttrFlags}). + * @param locations a file's block locations + */ + public LocatedFileStatus(long length, boolean isdir, int block_replication, + long blocksize, long modification_time, long access_time, + FsPermission permission, String owner, String group, Path symlink, + Path path, Set<AttrFlags> attr, BlockLocation[] locations) { + super(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, attr); this.locations = locations; } @@ -135,7 +162,16 @@ public class LocatedFileStatus extends FileStatus { public BlockLocation[] getBlockLocations() { return locations; } - + + /** + * Hook for subclasses to lazily set block locations. The {@link #locations} + * field should be null before this is called. + * @param locations Block locations for this instance. + */ + protected void setBlockLocations(BlockLocation[] locations) { + this.locations = locations; + } + /** * Compare this FileStatus to another FileStatus * @param o the FileStatus to be compared. http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java index 23caf2e..3b6724a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java @@ -96,12 +96,11 @@ public final class PBHelper { int flags = proto.getFlags(); FileStatus fileStatus = new FileStatus(length, isdir, blockReplication, blocksize, mtime, atime, permission, owner, group, symlink, path, - (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0, - (flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0, - (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0); - - fileStatus.setSnapShotEnabledFlag((flags & FileStatusProto.Flags - .SNAPSHOT_ENABLED_VALUE) != 0); + FileStatus.attributes( + (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0, + (flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0, + (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0, + (flags & FileStatusProto.Flags.SNAPSHOT_ENABLED_VALUE) != 0)); return fileStatus; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 9d6ab9a..22ef722 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -60,4 +60,34 @@ <Field name="cachingStrategy" /> <Bug pattern="IS2_INCONSISTENT_SYNC" /> </Match> + + <!-- BlockLocations are user-facing, but LocatedBlocks are not. --> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> + <Field name="hdfsloc" /> + <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" /> + </Match> + + <!-- Hdfs*FileStatus are internal types. This "internal" state is not sensitive. --> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsNamedFileStatus" /> + <Method name="getLocalNameInBytes" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsNamedFileStatus" /> + <Method name="getSymlinkInBytes" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> + <Method name="getLocalNameInBytes" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> + <Method name="getSymlinkInBytes" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 0499f2e..264e3f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -18,271 +18,40 @@ package org.apache.hadoop.hdfs.protocol; import java.io.IOException; +import java.io.ObjectInputValidation; +import java.io.Serializable; import java.net.URI; import java.util.Arrays; import java.util.EnumSet; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileStatus.AttrFlags; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.io.Writable; -/** Interface that represents the over the wire information for a file. +/** + * HDFS metadata for an entity in the filesystem. */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class HdfsFileStatus extends FileStatus { - - private static final long serialVersionUID = 0x126eb82a; - - // local name of the inode that's encoded in java UTF8 - private byte[] uPath; - private byte[] uSymlink; // symlink target encoded in java UTF8/null - private final long fileId; - private final FileEncryptionInfo feInfo; - private final ErasureCodingPolicy ecPolicy; +public interface HdfsFileStatus + extends Writable, Comparable<Object>, Serializable, ObjectInputValidation { - // Used by dir, not including dot and dotdot. Always zero for a regular file. - private final int childrenNum; - private final byte storagePolicy; + byte[] EMPTY_NAME = new byte[0]; - public static final byte[] EMPTY_NAME = new byte[0]; - - /** - * Set of features potentially active on an instance. - */ - public enum Flags { + /** Set of features potentially active on an instance. */ + enum Flags { HAS_ACL, HAS_CRYPT, HAS_EC, SNAPSHOT_ENABLED } - private final EnumSet<Flags> flags; - - /** - * Constructor. - * @param length the number of bytes the file has - * @param isdir if the path is a directory - * @param replication the replication factor - * @param blocksize the block size - * @param mtime modification time - * @param atime access time - * @param permission permission - * @param owner the owner of the path - * @param group the group of the path - * @param symlink symlink target encoded in java UTF8 or null - * @param path the local name in java UTF8 encoding the same as that in-memory - * @param fileId the file id - * @param childrenNum the number of children. Used by directory. - * @param feInfo the file's encryption info - * @param storagePolicy ID which specifies storage policy - * @param ecPolicy the erasure coding policy - */ - protected HdfsFileStatus(long length, boolean isdir, int replication, - long blocksize, long mtime, long atime, - FsPermission permission, EnumSet<Flags> flags, - String owner, String group, - byte[] symlink, byte[] path, long fileId, - int childrenNum, FileEncryptionInfo feInfo, - byte storagePolicy, ErasureCodingPolicy ecPolicy) { - super(length, isdir, replication, blocksize, mtime, - atime, convert(isdir, symlink != null, permission, flags), - owner, group, null, null, - flags.contains(Flags.HAS_ACL), flags.contains(Flags.HAS_CRYPT), - flags.contains(Flags.HAS_EC)); - this.flags = flags; - this.uSymlink = symlink; - this.uPath = path; - this.fileId = fileId; - this.childrenNum = childrenNum; - this.feInfo = feInfo; - this.storagePolicy = storagePolicy; - this.ecPolicy = ecPolicy; - } - - /** - * Set redundant flags for compatibility with existing applications. - */ - protected static FsPermission convert(boolean isdir, boolean symlink, - FsPermission p, EnumSet<Flags> f) { - if (p instanceof FsPermissionExtension) { - // verify flags are set consistently - assert p.getAclBit() == f.contains(HdfsFileStatus.Flags.HAS_ACL); - assert p.getEncryptedBit() == f.contains(HdfsFileStatus.Flags.HAS_CRYPT); - assert p.getErasureCodedBit() == f.contains(HdfsFileStatus.Flags.HAS_EC); - return p; - } - if (null == p) { - if (isdir) { - p = FsPermission.getDirDefault(); - } else if (symlink) { - p = FsPermission.getDefault(); - } else { - p = FsPermission.getFileDefault(); - } - } - return new FsPermissionExtension(p, f.contains(Flags.HAS_ACL), - f.contains(Flags.HAS_CRYPT), f.contains(Flags.HAS_EC)); - } - - @Override - public boolean isSymlink() { - return uSymlink != null; - } - - @Override - public boolean hasAcl() { - return flags.contains(Flags.HAS_ACL); - } - - @Override - public boolean isEncrypted() { - return flags.contains(Flags.HAS_CRYPT); - } - - @Override - public boolean isErasureCoded() { - return flags.contains(Flags.HAS_EC); - } - - /** - * Check if the local name is empty. - * @return true if the name is empty - */ - public final boolean isEmptyLocalName() { - return uPath.length == 0; - } - - /** - * Get the string representation of the local name. - * @return the local name in string - */ - public final String getLocalName() { - return DFSUtilClient.bytes2String(uPath); - } - - /** - * Get the Java UTF8 representation of the local name. - * @return the local name in java UTF8 - */ - public final byte[] getLocalNameInBytes() { - return uPath; - } - - /** - * Get the string representation of the full path name. - * @param parent the parent path - * @return the full path in string - */ - public final String getFullName(final String parent) { - if (isEmptyLocalName()) { - return parent; - } - - StringBuilder fullName = new StringBuilder(parent); - if (!parent.endsWith(Path.SEPARATOR)) { - fullName.append(Path.SEPARATOR); - } - fullName.append(getLocalName()); - return fullName.toString(); - } - - /** - * Get the full path. - * @param parent the parent path - * @return the full path - */ - public final Path getFullPath(final Path parent) { - if (isEmptyLocalName()) { - return parent; - } - - return new Path(parent, getLocalName()); - } - - @Override - public Path getSymlink() throws IOException { - if (isSymlink()) { - return new Path(DFSUtilClient.bytes2String(uSymlink)); - } - throw new IOException("Path " + getPath() + " is not a symbolic link"); - } - - @Override - public void setSymlink(Path sym) { - uSymlink = DFSUtilClient.string2Bytes(sym.toString()); - } - - /** - * Opaque referant for the symlink, to be resolved at the client. - */ - public final byte[] getSymlinkInBytes() { - return uSymlink; - } - - public final long getFileId() { - return fileId; - } - - public final FileEncryptionInfo getFileEncryptionInfo() { - return feInfo; - } - - /** - * Get the erasure coding policy if it's set. - * @return the erasure coding policy - */ - public ErasureCodingPolicy getErasureCodingPolicy() { - return ecPolicy; - } - - public final int getChildrenNum() { - return childrenNum; - } - - /** @return the storage policy id */ - public final byte getStoragePolicy() { - return storagePolicy; - } - - /** - * Check if directory is Snapshot enabled or not. - * - * @return true if directory is snapshot enabled - */ - public boolean isSnapshotEnabled() { - return flags.contains(Flags.SNAPSHOT_ENABLED); - } - - @Override - public boolean equals(Object o) { - // satisfy findbugs - return super.equals(o); - } - - @Override - public int hashCode() { - // satisfy findbugs - return super.hashCode(); - } - - /** - * Resolve the short name of the Path given the URI, parent provided. This - * FileStatus reference will not contain a valid Path until it is resolved - * by this method. - * @param defaultUri FileSystem to fully qualify HDFS path. - * @param parent Parent path of this element. - * @return Reference to this instance. - */ - public final FileStatus makeQualified(URI defaultUri, Path parent) { - // fully-qualify path - setPath(getFullPath(parent).makeQualified(defaultUri, null)); - return this; // API compatibility - - } /** * Builder class for HdfsFileStatus instances. Note default values for @@ -290,7 +59,7 @@ public class HdfsFileStatus extends FileStatus { */ @InterfaceAudience.Private @InterfaceStability.Unstable - public static class Builder { + class Builder { // Changing default values will affect cases where values are not // specified. Be careful! private long length = 0L; @@ -311,6 +80,7 @@ public class HdfsFileStatus extends FileStatus { private byte storagePolicy = HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; private ErasureCodingPolicy ecPolicy = null; + private LocatedBlocks locations = null; /** * Set the length of the entity (default = 0). @@ -490,13 +260,279 @@ public class HdfsFileStatus extends FileStatus { } /** + * Set the block locations for this entity (default = null). + * @param locations HDFS locations + * (see {@link HdfsLocatedFileStatus#makeQualifiedLocated(URI, Path)}) + * @return This Builder instance + */ + public Builder locations(LocatedBlocks locations) { + this.locations = locations; + return this; + } + + /** * @return An {@link HdfsFileStatus} instance from these parameters. */ public HdfsFileStatus build() { - return new HdfsFileStatus(length, isdir, replication, blocksize, - mtime, atime, permission, flags, owner, group, symlink, path, fileId, - childrenNum, feInfo, storagePolicy, ecPolicy); + if (null == locations && !isdir && null == symlink) { + return new HdfsNamedFileStatus(length, isdir, replication, blocksize, + mtime, atime, permission, flags, owner, group, symlink, path, + fileId, childrenNum, feInfo, storagePolicy, ecPolicy); + } + return new HdfsLocatedFileStatus(length, isdir, replication, blocksize, + mtime, atime, permission, flags, owner, group, symlink, path, + fileId, childrenNum, feInfo, storagePolicy, ecPolicy, locations); + } + + } + + /////////////////// + // HDFS-specific // + /////////////////// + + /** + * Inode ID for this entity, if a file. + * @return inode ID. + */ + long getFileId(); + + /** + * Get metadata for encryption, if present. + * @return the {@link FileEncryptionInfo} for this stream, or null if not + * encrypted. + */ + FileEncryptionInfo getFileEncryptionInfo(); + + /** + * Check if the local name is empty. + * @return true if the name is empty + */ + default boolean isEmptyLocalName() { + return getLocalNameInBytes().length == 0; + } + + /** + * Get the string representation of the local name. + * @return the local name in string + */ + default String getLocalName() { + return DFSUtilClient.bytes2String(getLocalNameInBytes()); + } + + /** + * Get the Java UTF8 representation of the local name. + * @return the local name in java UTF8 + */ + byte[] getLocalNameInBytes(); + + /** + * Get the string representation of the full path name. + * @param parent the parent path + * @return the full path in string + */ + default String getFullName(String parent) { + if (isEmptyLocalName()) { + return parent; + } + + StringBuilder fullName = new StringBuilder(parent); + if (!parent.endsWith(Path.SEPARATOR)) { + fullName.append(Path.SEPARATOR); + } + fullName.append(getLocalName()); + return fullName.toString(); + } + + /** + * Get the full path. + * @param parent the parent path + * @return the full path + */ + default Path getFullPath(Path parent) { + if (isEmptyLocalName()) { + return parent; + } + + return new Path(parent, getLocalName()); + } + + /** + * Opaque referant for the symlink, to be resolved at the client. + */ + byte[] getSymlinkInBytes(); + + /** + * @return number of children for this inode. + */ + int getChildrenNum(); + + /** + * Get the erasure coding policy if it's set. + * @return the erasure coding policy + */ + ErasureCodingPolicy getErasureCodingPolicy(); + + /** @return the storage policy id */ + byte getStoragePolicy(); + + /** + * Resolve the short name of the Path given the URI, parent provided. This + * FileStatus reference will not contain a valid Path until it is resolved + * by this method. + * @param defaultUri FileSystem to fully qualify HDFS path. + * @param parent Parent path of this element. + * @return Reference to this instance. + */ + default FileStatus makeQualified(URI defaultUri, Path parent) { + // fully-qualify path + setPath(getFullPath(parent).makeQualified(defaultUri, null)); + return (FileStatus) this; // API compatibility + } + + //////////////////////////// + // FileStatus "overrides" // + //////////////////////////// + + /** + * See {@link FileStatus#getPath()}. + */ + Path getPath(); + /** + * See {@link FileStatus#setPath(Path)}. + */ + void setPath(Path p); + /** + * See {@link FileStatus#getLen()}. + */ + long getLen(); + /** + * See {@link FileStatus#isFile()}. + */ + boolean isFile(); + /** + * See {@link FileStatus#isDirectory()}. + */ + boolean isDirectory(); + /** + * See {@link FileStatus#isDir()}. + */ + boolean isDir(); + /** + * See {@link FileStatus#isSymlink()}. + */ + boolean isSymlink(); + /** + * See {@link FileStatus#getBlockSize()}. + */ + long getBlockSize(); + /** + * See {@link FileStatus#getReplication()}. + */ + short getReplication(); + /** + * See {@link FileStatus#getModificationTime()}. + */ + long getModificationTime(); + /** + * See {@link FileStatus#getAccessTime()}. + */ + long getAccessTime(); + /** + * See {@link FileStatus#getPermission()}. + */ + FsPermission getPermission(); + /** + * See {@link FileStatus#setPermission(FsPermission)}. + */ + void setPermission(FsPermission permission); + /** + * See {@link FileStatus#getOwner()}. + */ + String getOwner(); + /** + * See {@link FileStatus#setOwner(String)}. + */ + void setOwner(String owner); + /** + * See {@link FileStatus#getGroup()}. + */ + String getGroup(); + /** + * See {@link FileStatus#setGroup(String)}. + */ + void setGroup(String group); + /** + * See {@link FileStatus#hasAcl()}. + */ + boolean hasAcl(); + /** + * See {@link FileStatus#isEncrypted()}. + */ + boolean isEncrypted(); + /** + * See {@link FileStatus#isErasureCoded()}. + */ + boolean isErasureCoded(); + /** + * See {@link FileStatus#isSnapshotEnabled()}. + */ + boolean isSnapshotEnabled(); + /** + * See {@link FileStatus#getSymlink()}. + */ + Path getSymlink() throws IOException; + /** + * See {@link FileStatus#setSymlink(Path sym)}. + */ + void setSymlink(Path sym); + /** + * See {@link FileStatus#compareTo(FileStatus)}. + */ + int compareTo(FileStatus stat); + + /** + * Set redundant flags for compatibility with existing applications. + */ + static FsPermission convert(boolean isdir, boolean symlink, + FsPermission p, Set<Flags> f) { + if (p instanceof FsPermissionExtension) { + // verify flags are set consistently + assert p.getAclBit() == f.contains(HdfsFileStatus.Flags.HAS_ACL); + assert p.getEncryptedBit() == f.contains(HdfsFileStatus.Flags.HAS_CRYPT); + assert p.getErasureCodedBit() == f.contains(HdfsFileStatus.Flags.HAS_EC); + return p; + } + if (null == p) { + if (isdir) { + p = FsPermission.getDirDefault(); + } else if (symlink) { + p = FsPermission.getDefault(); + } else { + p = FsPermission.getFileDefault(); + } + } + return new FsPermissionExtension(p, f.contains(Flags.HAS_ACL), + f.contains(Flags.HAS_CRYPT), f.contains(Flags.HAS_EC)); + } + + static Set<AttrFlags> convert(Set<Flags> flags) { + if (flags.isEmpty()) { + return FileStatus.NONE; + } + EnumSet<AttrFlags> attr = EnumSet.noneOf(AttrFlags.class); + if (flags.contains(Flags.HAS_ACL)) { + attr.add(AttrFlags.HAS_ACL); + } + if (flags.contains(Flags.HAS_EC)) { + attr.add(AttrFlags.HAS_EC); + } + if (flags.contains(Flags.HAS_CRYPT)) { + attr.add(AttrFlags.HAS_CRYPT); + } + if (flags.contains(Flags.SNAPSHOT_ENABLED)) { + attr.add(AttrFlags.SNAPSHOT_ENABLED); } + return attr; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 193aae2..1490e4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; +import java.io.IOException; import java.net.URI; import java.util.EnumSet; @@ -29,71 +30,150 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtilClient; /** - * Interface that represents the over the wire information - * including block locations for a file. + * HDFS metadata for an entity in the filesystem with locations. Note that + * symlinks and directories are returned as {@link HdfsLocatedFileStatus} for + * backwards compatibility. */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class HdfsLocatedFileStatus extends HdfsFileStatus { +public class HdfsLocatedFileStatus + extends LocatedFileStatus implements HdfsFileStatus { + private static final long serialVersionUID = 0x126eb82a; - private static final long serialVersionUID = 0x23c73328; + // local name of the inode that's encoded in java UTF8 + private byte[] uPath; + private byte[] uSymlink; // symlink target encoded in java UTF8/null + private final long fileId; + private final FileEncryptionInfo feInfo; + private final ErasureCodingPolicy ecPolicy; - /** - * Left transient, because {@link #makeQualifiedLocated(URI,Path)} - * is the user-facing type. - */ - private transient LocatedBlocks locations; + // Used by dir, not including dot and dotdot. Always zero for a regular file. + private final int childrenNum; + private final byte storagePolicy; + + // BlockLocations[] is the user-facing type + private transient LocatedBlocks hdfsloc; /** - * Constructor - * - * @param length size - * @param isdir if this is directory - * @param block_replication the file's replication factor - * @param blocksize the file's block size - * @param modification_time most recent modification time - * @param access_time most recent access time + * Constructor. + * @param length the number of bytes the file has + * @param isdir if the path is a directory + * @param replication the replication factor + * @param blocksize the block size + * @param mtime modification time + * @param atime access time * @param permission permission - * @param owner owner - * @param group group - * @param symlink symbolic link - * @param path local path name in java UTF8 format + * @param owner the owner of the path + * @param group the group of the path + * @param symlink symlink target encoded in java UTF8 or null + * @param path the local name in java UTF8 encoding the same as that in-memory * @param fileId the file id - * @param locations block locations - * @param feInfo file encryption info + * @param childrenNum the number of children. Used by directory. + * @param feInfo the file's encryption info + * @param storagePolicy ID which specifies storage policy + * @param ecPolicy the erasure coding policy + * @param hdfsloc block locations */ - public HdfsLocatedFileStatus(long length, boolean isdir, - int block_replication, long blocksize, long modification_time, - long access_time, FsPermission permission, EnumSet<Flags> flags, - String owner, String group, byte[] symlink, byte[] path, long fileId, - LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo, - byte storagePolicy, ErasureCodingPolicy ecPolicy) { - super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, flags, owner, group, symlink, path, fileId, - childrenNum, feInfo, storagePolicy, ecPolicy); - this.locations = locations; + HdfsLocatedFileStatus(long length, boolean isdir, int replication, + long blocksize, long mtime, long atime, + FsPermission permission, EnumSet<Flags> flags, + String owner, String group, + byte[] symlink, byte[] path, long fileId, + int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy, ErasureCodingPolicy ecPolicy, + LocatedBlocks hdfsloc) { + super(length, isdir, replication, blocksize, mtime, atime, + HdfsFileStatus.convert(isdir, symlink != null, permission, flags), + owner, group, null, null, HdfsFileStatus.convert(flags), + null); + this.uSymlink = symlink; + this.uPath = path; + this.fileId = fileId; + this.childrenNum = childrenNum; + this.feInfo = feInfo; + this.storagePolicy = storagePolicy; + this.ecPolicy = ecPolicy; + this.hdfsloc = hdfsloc; } - public LocatedBlocks getBlockLocations() { - return locations; + @Override // visibility + public void setOwner(String owner) { + super.setOwner(owner); + } + + @Override // visibility + public void setGroup(String group) { + super.setOwner(group); + } + + @Override + public boolean isSymlink() { + return uSymlink != null; + } + + @Override + public Path getSymlink() throws IOException { + if (isSymlink()) { + return new Path(DFSUtilClient.bytes2String(getSymlinkInBytes())); + } + throw new IOException("Path " + getPath() + " is not a symbolic link"); + } + + @Override // visibility + public void setPermission(FsPermission permission) { + super.setPermission(permission); } /** - * This function is used to transform the underlying HDFS LocatedBlocks to - * BlockLocations. - * - * The returned BlockLocation will have different formats for replicated - * and erasure coded file. - * Please refer to - * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations - * (FileStatus, long, long)} - * for examples. + * Get the Java UTF8 representation of the local name. + * @return the local name in java UTF8 */ - public final LocatedFileStatus makeQualifiedLocated(URI defaultUri, - Path path) { - makeQualified(defaultUri, path); - return new LocatedFileStatus(this, - DFSUtilClient.locatedBlocks2Locations(getBlockLocations())); + @Override + public byte[] getLocalNameInBytes() { + return uPath; + } + + @Override + public void setSymlink(Path sym) { + uSymlink = DFSUtilClient.string2Bytes(sym.toString()); + } + + /** + * Opaque referant for the symlink, to be resolved at the client. + */ + @Override + public byte[] getSymlinkInBytes() { + return uSymlink; + } + + @Override + public long getFileId() { + return fileId; + } + + @Override + public FileEncryptionInfo getFileEncryptionInfo() { + return feInfo; + } + + /** + * Get the erasure coding policy if it's set. + * @return the erasure coding policy + */ + @Override + public ErasureCodingPolicy getErasureCodingPolicy() { + return ecPolicy; + } + + @Override + public int getChildrenNum() { + return childrenNum; + } + + /** @return the storage policy id */ + @Override + public byte getStoragePolicy() { + return storagePolicy; } @Override @@ -107,4 +187,34 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { // satisfy findbugs return super.hashCode(); } + + /** + * Get block locations for this entity, in HDFS format. + * See {@link #makeQualifiedLocated(URI, Path)}. + * See {@link DFSUtilClient#locatedBlocks2Locations(LocatedBlocks)}. + * @return block locations + */ + public LocatedBlocks getLocatedBlocks() { + return hdfsloc; + } + + /** + * This function is used to transform the underlying HDFS LocatedBlocks to + * BlockLocations. This method must be invoked before + * {@link #getBlockLocations()}. + * + * The returned BlockLocation will have different formats for replicated + * and erasure coded file. + * Please refer to + * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations + * (FileStatus, long, long)} + * for examples. + */ + public LocatedFileStatus makeQualifiedLocated(URI defaultUri, Path path) { + makeQualified(defaultUri, path); + setBlockLocations( + DFSUtilClient.locatedBlocks2Locations(getLocatedBlocks())); + return this; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java new file mode 100644 index 0000000..311f9d0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSUtilClient; + +import java.io.IOException; +import java.util.Set; + +/** + * HDFS metadata for an entity in the filesystem without locations. Note that + * symlinks and directories are returned as {@link HdfsLocatedFileStatus} for + * backwards compatibility. + */ +public class HdfsNamedFileStatus extends FileStatus implements HdfsFileStatus { + + // local name of the inode that's encoded in java UTF8 + private byte[] uPath; + private byte[] uSymlink; // symlink target encoded in java UTF8/null + private final long fileId; + private final FileEncryptionInfo feInfo; + private final ErasureCodingPolicy ecPolicy; + + // Used by dir, not including dot and dotdot. Always zero for a regular file. + private final int childrenNum; + private final byte storagePolicy; + + /** + * Constructor. + * @param length the number of bytes the file has + * @param isdir if the path is a directory + * @param replication the replication factor + * @param blocksize the block size + * @param mtime modification time + * @param atime access time + * @param permission permission + * @param owner the owner of the path + * @param group the group of the path + * @param symlink symlink target encoded in java UTF8 or null + * @param path the local name in java UTF8 encoding the same as that in-memory + * @param fileId the file id + * @param childrenNum the number of children. Used by directory. + * @param feInfo the file's encryption info + * @param storagePolicy ID which specifies storage policy + * @param ecPolicy the erasure coding policy + */ + HdfsNamedFileStatus(long length, boolean isdir, int replication, + long blocksize, long mtime, long atime, + FsPermission permission, Set<Flags> flags, + String owner, String group, + byte[] symlink, byte[] path, long fileId, + int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy, ErasureCodingPolicy ecPolicy) { + super(length, isdir, replication, blocksize, mtime, atime, + HdfsFileStatus.convert(isdir, symlink != null, permission, flags), + owner, group, null, null, + HdfsFileStatus.convert(flags)); + this.uSymlink = symlink; + this.uPath = path; + this.fileId = fileId; + this.childrenNum = childrenNum; + this.feInfo = feInfo; + this.storagePolicy = storagePolicy; + this.ecPolicy = ecPolicy; + } + + @Override + public void setOwner(String owner) { + super.setOwner(owner); + } + + @Override + public void setGroup(String group) { + super.setOwner(group); + } + + @Override + public boolean isSymlink() { + return uSymlink != null; + } + + @Override + public Path getSymlink() throws IOException { + if (isSymlink()) { + return new Path(DFSUtilClient.bytes2String(getSymlinkInBytes())); + } + throw new IOException("Path " + getPath() + " is not a symbolic link"); + } + + @Override + public void setPermission(FsPermission permission) { + super.setPermission(permission); + } + + /** + * Get the Java UTF8 representation of the local name. + * + * @return the local name in java UTF8 + */ + @Override + public byte[] getLocalNameInBytes() { + return uPath; + } + + @Override + public void setSymlink(Path sym) { + uSymlink = DFSUtilClient.string2Bytes(sym.toString()); + } + + /** + * Opaque referant for the symlink, to be resolved at the client. + */ + @Override + public byte[] getSymlinkInBytes() { + return uSymlink; + } + + @Override + public long getFileId() { + return fileId; + } + + @Override + public FileEncryptionInfo getFileEncryptionInfo() { + return feInfo; + } + + /** + * Get the erasure coding policy if it's set. + * + * @return the erasure coding policy + */ + @Override + public ErasureCodingPolicy getErasureCodingPolicy() { + return ecPolicy; + } + + @Override + public int getChildrenNum() { + return childrenNum; + } + + /** @return the storage policy id */ + @Override + public byte getStoragePolicy() { + return storagePolicy; + } + + @Override + public boolean equals(Object o) { + // satisfy findbugs + return super.equals(o); + } + + @Override + public int hashCode() { + // satisfy findbugs + return super.hashCode(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/package-info.java new file mode 100644 index 0000000..e1a8ceb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index ae3921c..d3b7f6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1585,23 +1585,36 @@ public class PBHelperClient { EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags() ? convertFlags(fs.getFlags()) : convertFlags(fs.getPermission()); - return new HdfsLocatedFileStatus( - fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), - fs.getBlockReplication(), fs.getBlocksize(), - fs.getModificationTime(), fs.getAccessTime(), - convert(fs.getPermission()), - flags, - fs.getOwner(), fs.getGroup(), - fs.getFileType().equals(FileType.IS_SYMLINK) ? - fs.getSymlink().toByteArray() : null, - fs.getPath().toByteArray(), - fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID, - fs.hasLocations() ? convert(fs.getLocations()) : null, - fs.hasChildrenNum() ? fs.getChildrenNum() : -1, - fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, - fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() - : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, - fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null); + return new HdfsFileStatus.Builder() + .length(fs.getLength()) + .isdir(fs.getFileType().equals(FileType.IS_DIR)) + .replication(fs.getBlockReplication()) + .blocksize(fs.getBlocksize()) + .mtime(fs.getModificationTime()) + .atime(fs.getAccessTime()) + .perm(convert(fs.getPermission())) + .flags(flags) + .owner(fs.getOwner()) + .group(fs.getGroup()) + .symlink(FileType.IS_SYMLINK.equals(fs.getFileType()) + ? fs.getSymlink().toByteArray() + : null) + .path(fs.getPath().toByteArray()) + .fileId(fs.hasFileId() + ? fs.getFileId() + : HdfsConstants.GRANDFATHER_INODE_ID) + .locations(fs.hasLocations() ? convert(fs.getLocations()) : null) + .children(fs.hasChildrenNum() ? fs.getChildrenNum() : -1) + .feInfo(fs.hasFileEncryptionInfo() + ? convert(fs.getFileEncryptionInfo()) + : null) + .storagePolicy(fs.hasStoragePolicy() + ? (byte) fs.getStoragePolicy() + : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) + .ecPolicy(fs.hasEcPolicy() + ? convertErasureCodingPolicy(fs.getEcPolicy()) + : null) + .build(); } private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) { @@ -1864,10 +1877,10 @@ public class PBHelperClient { if (dl == null) return null; List<HdfsFileStatusProto> partList = dl.getPartialListingList(); - return new DirectoryListing(partList.isEmpty() ? - new HdfsLocatedFileStatus[0] : - convert(partList.toArray(new HdfsFileStatusProto[partList.size()])), - dl.getRemainingEntries()); + return new DirectoryListing(partList.isEmpty() + ? new HdfsFileStatus[0] + : convert(partList.toArray(new HdfsFileStatusProto[partList.size()])), + dl.getRemainingEntries()); } public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) { @@ -2163,7 +2176,7 @@ public class PBHelperClient { } if (fs instanceof HdfsLocatedFileStatus) { final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs; - LocatedBlocks locations = lfs.getBlockLocations(); + LocatedBlocks locations = lfs.getLocatedBlocks(); if (locations != null) { builder.setLocations(convert(locations)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsFileStatusMethods.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsFileStatusMethods.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsFileStatusMethods.java new file mode 100644 index 0000000..3cc4190 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestHdfsFileStatusMethods.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Stream; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toSet; + +import org.apache.hadoop.fs.FileStatus; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit test verifying that {@link HdfsFileStatus} is a superset of + * {@link FileStatus}. + */ +public class TestHdfsFileStatusMethods { + + @Test + public void testInterfaceSuperset() { + Set<MethodSignature> fsM = signatures(FileStatus.class); + Set<MethodSignature> hfsM = signatures(HdfsFileStatus.class); + hfsM.addAll(Stream.of(HdfsFileStatus.class.getInterfaces()) + .flatMap(i -> Stream.of(i.getDeclaredMethods())) + .map(MethodSignature::new) + .collect(toSet())); + // HdfsFileStatus is not a concrete type + hfsM.addAll(signatures(Object.class)); + assertTrue(fsM.removeAll(hfsM)); + // verify that FileStatus is a subset of HdfsFileStatus + assertEquals(fsM.stream() + .map(MethodSignature::toString) + .collect(joining("\n")), + Collections.EMPTY_SET, fsM); + } + + /** Map non-static, declared methods for this class to signatures. */ + private static Set<MethodSignature> signatures(Class<?> c) { + return Stream.of(c.getDeclaredMethods()) + .filter(m -> !Modifier.isStatic(m.getModifiers())) + .map(MethodSignature::new) + .collect(toSet()); + } + + private static class MethodSignature { + private final String name; + private final Type rval; + private final Type[] param; + MethodSignature(Method m) { + name = m.getName(); + rval = m.getGenericReturnType(); + param = m.getParameterTypes(); + } + @Override + public int hashCode() { + return name.hashCode(); + } + /** + * Methods are equal iff they have the same name, return type, and params + * (non-generic). + */ + @Override + public boolean equals(Object o) { + if (!(o instanceof MethodSignature)) { + return false; + } + MethodSignature s = (MethodSignature) o; + return name.equals(s.name) && + rval.equals(s.rval) && + Arrays.equals(param, s.param); + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(rval).append(" ").append(name).append("(") + .append(Stream.of(param) + .map(Type::toString).collect(joining(","))) + .append(")"); + return sb.toString(); + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index ffd64a1..54eaa78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -1085,10 +1085,7 @@ public class HttpFSFileSystem extends FileSystem new FsPermissionExtension(permission, aBit, eBit, ecBit); FileStatus fileStatus = new FileStatus(len, FILE_TYPE.DIRECTORY == type, replication, blockSize, mTime, aTime, deprecatedPerm, owner, group, - null, path, aBit, eBit, ecBit); - if (seBit) { - fileStatus.setSnapShotEnabledFlag(seBit); - } + null, path, FileStatus.attributes(aBit, eBit, ecBit, seBit)); return fileStatus; } else { return new FileStatus(len, FILE_TYPE.DIRECTORY == type, http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml index 4b958b5..a99ea75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml @@ -260,13 +260,6 @@ <Method name="visitFile" /> <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" /> </Match> - <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not. - Defensible compatibility choices over time create odd corners. --> - <Match> - <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> - <Field name="locations" /> - <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" /> - </Match> <Match> <Class name="org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil$1" /> <Method name="visitFile" /> http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index b653f4f..8b89378 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -395,7 +395,7 @@ public class Mover { status.getReplication()); final ErasureCodingPolicy ecPolicy = status.getErasureCodingPolicy(); - final LocatedBlocks locatedBlocks = status.getBlockLocations(); + final LocatedBlocks locatedBlocks = status.getLocatedBlocks(); final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete(); List<LocatedBlock> lbs = locatedBlocks.getLocatedBlocks(); for (int i = 0; i < lbs.size(); i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index b6e38a7..4da8a28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -261,7 +261,7 @@ class FSDirStatAndListingOp { // This helps to prevent excessively large response payloads. // Approximate #locations with locatedBlockCount() * repl_factor LocatedBlocks blks = - ((HdfsLocatedFileStatus)listing[i]).getBlockLocations(); + ((HdfsLocatedFileStatus)listing[i]).getLocatedBlocks(); locationBudget -= (blks == null) ? 0 : blks.locatedBlockCount() * listing[i].getReplication(); } @@ -486,31 +486,26 @@ class FSDirStatAndListingOp { String owner, String group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy, ErasureCodingPolicy ecPolicy, LocatedBlocks locations) { - if (locations == null) { - return new HdfsFileStatus.Builder() - .length(length) - .isdir(isdir) - .replication(replication) - .blocksize(blocksize) - .mtime(mtime) - .atime(atime) - .perm(permission) - .flags(flags) - .owner(owner) - .group(group) - .symlink(symlink) - .path(path) - .fileId(fileId) - .children(childrenNum) - .feInfo(feInfo) - .storagePolicy(storagePolicy) - .ecPolicy(ecPolicy) - .build(); - } else { - return new HdfsLocatedFileStatus(length, isdir, replication, blocksize, - mtime, atime, permission, flags, owner, group, symlink, path, - fileId, locations, childrenNum, feInfo, storagePolicy, ecPolicy); - } + return new HdfsFileStatus.Builder() + .length(length) + .isdir(isdir) + .replication(replication) + .blocksize(blocksize) + .mtime(mtime) + .atime(atime) + .perm(permission) + .flags(flags) + .owner(owner) + .group(group) + .symlink(symlink) + .path(path) + .fileId(fileId) + .children(childrenNum) + .feInfo(feInfo) + .storagePolicy(storagePolicy) + .ecPolicy(ecPolicy) + .locations(locations) + .build(); } private static ContentSummary getContentSummaryInt(FSDirectory fsd, http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 3a8fb59..ae256a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -1096,7 +1096,7 @@ public class TestBlockStoragePolicy { int replicaNum, StorageType... types) { List<StorageType> typeList = Lists.newArrayList(); Collections.addAll(typeList, types); - LocatedBlocks lbs = status.getBlockLocations(); + LocatedBlocks lbs = status.getLocatedBlocks(); Assert.assertEquals(blockNum, lbs.getLocatedBlocks().size()); for (LocatedBlock lb : lbs.getLocatedBlocks()) { Assert.assertEquals(replicaNum, lb.getStorageTypes().length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java index 42c9acf..62d1b9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java @@ -110,7 +110,7 @@ public class TestFileStatusSerialization { dib.reset(dob.getData(), 0, dob.getLength()); FileStatus fstat = new FileStatus(); fstat.readFields(dib); - checkFields(stat, fstat); + checkFields((FileStatus) stat, fstat); // FsPermisisonExtension used for HdfsFileStatus, not FileStatus, // attribute flags should still be preserved @@ -133,7 +133,7 @@ public class TestFileStatusSerialization { try (ObjectInputStream ois = new ObjectInputStream(bais)) { FileStatus deser = (FileStatus) ois.readObject(); assertEquals(hs, deser); - checkFields(hs, deser); + checkFields((FileStatus) hs, deser); } } @@ -168,8 +168,8 @@ public class TestFileStatusSerialization { byte[] dst = fsp.toByteArray(); HdfsFileStatusProto hsp2 = HdfsFileStatusProto.parseFrom(dst); assertEquals(hsp, hsp2); - FileStatus hstat = PBHelperClient.convert(hsp); - FileStatus hstat2 = PBHelperClient.convert(hsp2); + FileStatus hstat = (FileStatus) PBHelperClient.convert(hsp); + FileStatus hstat2 = (FileStatus) PBHelperClient.convert(hsp2); checkFields(hstat, hstat2); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e560f3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index 764a0db..356ae3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -320,7 +320,7 @@ public class TestStorageMover { } final List<StorageType> types = policy.chooseStorageTypes( status.getReplication()); - for(LocatedBlock lb : fileStatus.getBlockLocations().getLocatedBlocks()) { + for(LocatedBlock lb : fileStatus.getLocatedBlocks().getLocatedBlocks()) { final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.getStorageTypes()); Assert.assertTrue(fileStatus.getFullName(parent.toString()) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
