HDFS-6984. Serialize FileStatus via protobuf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12e44e7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12e44e7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12e44e7b Branch: refs/heads/trunk Commit: 12e44e7bdaf53d3720a89d32f0cc2717241bd6b2 Parents: 1a1bf6b Author: Chris Douglas <[email protected]> Authored: Wed Aug 2 12:12:48 2017 -0700 Committer: Chris Douglas <[email protected]> Committed: Wed Aug 2 12:12:48 2017 -0700 ---------------------------------------------------------------------- .../dev-support/findbugsExcludeFile.xml | 4 + hadoop-common-project/hadoop-common/pom.xml | 1 + .../apache/hadoop/fs/FileEncryptionInfo.java | 6 +- .../java/org/apache/hadoop/fs/FileStatus.java | 152 ++++++++----- .../org/apache/hadoop/fs/LocatedFileStatus.java | 64 ++++-- .../hadoop/fs/permission/FsPermission.java | 12 + .../apache/hadoop/fs/protocolPB/PBHelper.java | 131 +++++++++++ .../hadoop/fs/protocolPB/package-info.java | 18 ++ .../apache/hadoop/io/erasurecode/ECSchema.java | 6 +- .../hadoop-common/src/main/proto/FSProtos.proto | 69 ++++++ .../org/apache/hadoop/fs/TestFileStatus.java | 1 + .../fs/protocolPB/TestFSSerialization.java | 85 +++++++ .../hdfs/protocol/ErasureCodingPolicy.java | 8 +- .../hdfs/protocol/FsPermissionExtension.java | 5 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 228 +++++++++---------- .../hdfs/protocol/HdfsLocatedFileStatus.java | 42 ++-- .../protocol/SnapshottableDirectoryStatus.java | 10 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +++++- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 50 ++-- .../hadoop/hdfs/web/WebHdfsConstants.java | 8 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +- .../hadoop-hdfs-client/src/main/proto/acl.proto | 7 +- .../src/main/proto/hdfs.proto | 16 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 45 ++-- .../fs/http/client/BaseTestHttpFSWith.java | 19 +- .../dev-support/findbugsExcludeFile.xml | 12 + .../hadoop/hdfs/protocol/SnapshotInfo.java | 2 +- .../hadoop/hdfs/protocolPB/package-info.java | 18 ++ .../server/namenode/FSDirStatAndListingOp.java | 58 ++--- .../hdfs/server/namenode/FSDirectory.java | 20 +- .../hdfs/server/namenode/FSNamesystem.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 3 +- .../namenode/snapshot/SnapshotManager.java | 7 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 28 ++- .../hadoop/hdfs/TestDFSClientRetries.java | 6 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 3 +- .../hdfs/TestFileStatusSerialization.java | 153 +++++++++++++ .../java/org/apache/hadoop/hdfs/TestLease.java | 7 +- .../hdfs/server/namenode/AclTestHelpers.java | 13 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 11 +- 41 files changed, 1054 insertions(+), 366 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index de76afb..4bafd8e 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -323,6 +323,10 @@ <!-- protobuf generated code --> <Class name="~org\.apache\.hadoop\.tracing\.TraceAdminPB.*"/> </Match> + <Match> + <!-- protobuf generated code --> + <Class name="~org\.apache\.hadoop\.fs\.FSProto.*"/> + </Match> <!-- Manually checked, misses child thread manually syncing on parent's intrinsic lock. http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 3e73bce..ad6f734 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -393,6 +393,7 @@ <include>RefreshUserMappingsProtocol.proto</include> <include>RefreshCallQueueProtocol.proto</include> <include>GenericRefreshProtocol.proto</include> + <include>FSProtos.proto</include> </includes> </source> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java index 1129e07..ce5ed56 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.fs; +import java.io.Serializable; + import org.apache.commons.codec.binary.Hex; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CipherSuite; @@ -30,7 +32,9 @@ import static com.google.common.base.Preconditions.checkNotNull; * an encrypted file. */ @InterfaceAudience.Private -public class FileEncryptionInfo { +public class FileEncryptionInfo implements Serializable { + + private static final long serialVersionUID = 0x156abe03; private final CipherSuite cipherSuite; private final CryptoProtocolVersion version; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 7ce6363..2f22ea0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -23,11 +23,15 @@ import java.io.IOException; import java.io.InvalidObjectException; import java.io.ObjectInputValidation; import java.io.Serializable; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; +import org.apache.hadoop.fs.FSProtos.FileStatusProto; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.Text; +import org.apache.hadoop.fs.protocolPB.PBHelper; import org.apache.hadoop.io.Writable; /** Interface that represents the client side information for a file. @@ -50,7 +54,31 @@ public class FileStatus implements Writable, Comparable<Object>, private String owner; private String group; private Path symlink; - + private Set<AttrFlags> attr; + + private enum AttrFlags { + HAS_ACL, + HAS_CRYPT, + HAS_EC, + }; + private static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet(); + private static Set<AttrFlags> flags(boolean acl, boolean crypt, boolean ec) { + if (!(acl || crypt || ec)) { + return NONE; + } + EnumSet<AttrFlags> ret = EnumSet.noneOf(AttrFlags.class); + if (acl) { + ret.add(AttrFlags.HAS_ACL); + } + if (crypt) { + ret.add(AttrFlags.HAS_CRYPT); + } + if (ec) { + ret.add(AttrFlags.HAS_EC); + } + return ret; + } + public FileStatus() { this(0, false, 0, 0, 0, 0, null, null, null, null); } //We should deprecate this soon? @@ -79,6 +107,15 @@ public class FileStatus implements Writable, Comparable<Object>, FsPermission permission, String owner, String group, Path symlink, Path path) { + this(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, + false, false, false); + } + + public FileStatus(long length, boolean isdir, int block_replication, + long blocksize, long modification_time, long access_time, + FsPermission permission, String owner, String group, Path symlink, + Path path, boolean hasAcl, boolean isEncrypted, boolean isErasureCoded) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -89,7 +126,7 @@ public class FileStatus implements Writable, Comparable<Object>, this.permission = permission; } else if (isdir) { this.permission = FsPermission.getDirDefault(); - } else if (symlink!=null) { + } else if (symlink != null) { this.permission = FsPermission.getDefault(); } else { this.permission = FsPermission.getFileDefault(); @@ -98,6 +135,8 @@ public class FileStatus implements Writable, Comparable<Object>, this.group = (group == null) ? "" : group; this.symlink = symlink; this.path = path; + attr = flags(hasAcl, isEncrypted, isErasureCoded); + // The variables isdir and symlink indicate the type: // 1. isdir implies directory, in which case symlink must be null. // 2. !isdir implies a file or symlink, symlink != null implies a @@ -213,7 +252,7 @@ public class FileStatus implements Writable, Comparable<Object>, * @return true if the underlying file or directory has ACLs set. */ public boolean hasAcl() { - return permission.getAclBit(); + return attr.contains(AttrFlags.HAS_ACL); } /** @@ -222,7 +261,7 @@ public class FileStatus implements Writable, Comparable<Object>, * @return true if the underlying file is encrypted. */ public boolean isEncrypted() { - return permission.getEncryptedBit(); + return attr.contains(AttrFlags.HAS_CRYPT); } /** @@ -231,7 +270,7 @@ public class FileStatus implements Writable, Comparable<Object>, * @return true if the underlying file or directory is erasure coded. */ public boolean isErasureCoded() { - return permission.getErasureCodedBit(); + return attr.contains(AttrFlags.HAS_EC); } /** @@ -304,47 +343,6 @@ public class FileStatus implements Writable, Comparable<Object>, public void setSymlink(final Path p) { symlink = p; } - - ////////////////////////////////////////////////// - // Writable - ////////////////////////////////////////////////// - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); - out.writeLong(getLen()); - out.writeBoolean(isDirectory()); - out.writeShort(getReplication()); - out.writeLong(getBlockSize()); - out.writeLong(getModificationTime()); - out.writeLong(getAccessTime()); - getPermission().write(out); - Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN); - Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN); - out.writeBoolean(isSymlink()); - if (isSymlink()) { - Text.writeString(out, getSymlink().toString(), Text.DEFAULT_MAX_LEN); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - String strPath = Text.readString(in, Text.DEFAULT_MAX_LEN); - this.path = new Path(strPath); - this.length = in.readLong(); - this.isdir = in.readBoolean(); - this.block_replication = in.readShort(); - blocksize = in.readLong(); - modification_time = in.readLong(); - access_time = in.readLong(); - permission.readFields(in); - owner = Text.readString(in, Text.DEFAULT_MAX_LEN); - group = Text.readString(in, Text.DEFAULT_MAX_LEN); - if (in.readBoolean()) { - this.symlink = new Path(Text.readString(in, Text.DEFAULT_MAX_LEN)); - } else { - this.symlink = null; - } - } /** * Compare this FileStatus to another FileStatus @@ -377,15 +375,12 @@ public class FileStatus implements Writable, Comparable<Object>, */ @Override public boolean equals(Object o) { - if (o == null) { + if (!(o instanceof FileStatus)) { return false; } if (this == o) { return true; } - if (!(o instanceof FileStatus)) { - return false; - } FileStatus other = (FileStatus)o; return this.getPath().equals(other.getPath()); } @@ -420,7 +415,11 @@ public class FileStatus implements Writable, Comparable<Object>, sb.append("; permission=" + permission); sb.append("; isSymlink=" + isSymlink()); if(isSymlink()) { - sb.append("; symlink=" + symlink); + try { + sb.append("; symlink=" + getSymlink()); + } catch (IOException e) { + throw new RuntimeException("Unexpected exception", e); + } } sb.append("; hasAcl=" + hasAcl()); sb.append("; isEncrypted=" + isEncrypted()); @@ -429,6 +428,55 @@ public class FileStatus implements Writable, Comparable<Object>, return sb.toString(); } + /** + * Read instance encoded as protobuf from stream. + * @param in Input stream + * @see PBHelper#convert(FileStatus) + * @deprecated Use the {@link PBHelper} and protobuf serialization directly. + */ + @Override + @Deprecated + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + if (size < 0) { + throw new IOException("Can't read FileStatusProto with negative " + + "size of " + size); + } + byte[] buf = new byte[size]; + in.readFully(buf); + FileStatusProto proto = FileStatusProto.parseFrom(buf); + FileStatus other = PBHelper.convert(proto); + isdir = other.isDirectory(); + length = other.getLen(); + isdir = other.isDirectory(); + block_replication = other.getReplication(); + blocksize = other.getBlockSize(); + modification_time = other.getModificationTime(); + access_time = other.getAccessTime(); + setPermission(other.getPermission()); + setOwner(other.getOwner()); + setGroup(other.getGroup()); + setSymlink((other.isSymlink() ? other.getSymlink() : null)); + setPath(other.getPath()); + attr = flags(other.hasAcl(), other.isEncrypted(), other.isErasureCoded()); + assert (isDirectory() && getSymlink() == null) || !isDirectory(); + } + + /** + * Write instance encoded as protobuf to stream. + * @param out Output stream + * @see PBHelper#convert(FileStatus) + * @deprecated Use the {@link PBHelper} and protobuf serialization directly. + */ + @Override + @Deprecated + public void write(DataOutput out) throws IOException { + FileStatusProto proto = PBHelper.convert(this); + int size = proto.getSerializedSize(); + out.writeInt(size); + out.write(proto.toByteArray()); + } + @Override public void validateObject() throws InvalidObjectException { if (null == path) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java index 588fd6a..885da07 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java @@ -30,6 +30,9 @@ import org.apache.hadoop.fs.permission.FsPermission; @InterfaceAudience.Public @InterfaceStability.Evolving public class LocatedFileStatus extends FileStatus { + + private static final long serialVersionUID = 0x17339920; + private BlockLocation[] locations; @@ -42,14 +45,18 @@ public class LocatedFileStatus extends FileStatus { * @param stat a file status * @param locations a file's block locations */ - public LocatedFileStatus(FileStatus stat, BlockLocation[] locations) - throws IOException { + public LocatedFileStatus(FileStatus stat, BlockLocation[] locations) { this(stat.getLen(), stat.isDirectory(), stat.getReplication(), stat.getBlockSize(), stat.getModificationTime(), - stat.getAccessTime(), stat.getPermission(), stat.getOwner(), - stat.getGroup(), null, stat.getPath(), locations); + stat.getAccessTime(), stat.getPermission(), + stat.getOwner(), stat.getGroup(), null, stat.getPath(), + stat.hasAcl(), stat.isEncrypted(), stat.isErasureCoded(), locations); if (stat.isSymlink()) { - setSymlink(stat.getSymlink()); + try { + setSymlink(stat.getSymlink()); + } catch (IOException e) { + throw new RuntimeException("Unexpected exception", e); + } } } @@ -69,24 +76,55 @@ public class LocatedFileStatus extends FileStatus { * @param path the path's qualified name * @param locations a file's block locations */ + @Deprecated public LocatedFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, - Path symlink, - Path path, - BlockLocation[] locations) { - super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, owner, group, symlink, path); - this.locations = locations; + Path symlink, Path path, BlockLocation[] locations) { + this(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, + permission.getAclBit(), permission.getEncryptedBit(), + permission.getErasureCodedBit(), locations); } - + + /** + * Constructor. + * + * @param length a file's length + * @param isdir if the path is a directory + * @param block_replication the file's replication factor + * @param blocksize a file's block size + * @param modification_time a file's modification time + * @param access_time a file's access time + * @param permission a file's permission + * @param owner a file's owner + * @param group a file's group + * @param symlink symlink if the path is a symbolic link + * @param path the path's qualified name + * @param hasAcl entity has associated ACLs + * @param isEncrypted entity is encrypted + * @param isErasureCoded entity is erasure coded + * @param locations a file's block locations + */ + public LocatedFileStatus(long length, boolean isdir, + int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, String owner, String group, + Path symlink, Path path, + boolean hasAcl, boolean isEncrypted, boolean isErasureCoded, + BlockLocation[] locations) { + super(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, + hasAcl, isEncrypted, isErasureCoded); + this.locations = locations; + } + /** * Get the file's block locations * @return the file's block locations */ public BlockLocation[] getBlockLocations() { - return locations; + return locations; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java index 73ab5f6..23692de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java @@ -133,11 +133,13 @@ public class FsPermission implements Writable, Serializable, } @Override + @Deprecated public void write(DataOutput out) throws IOException { out.writeShort(toShort()); } @Override + @Deprecated public void readFields(DataInput in) throws IOException { fromShort(in.readShort()); } @@ -184,6 +186,7 @@ public class FsPermission implements Writable, Serializable, * * @return short extended short representation of this permission */ + @Deprecated public short toExtendedShort() { return toShort(); } @@ -299,7 +302,10 @@ public class FsPermission implements Writable, Serializable, * Returns true if there is also an ACL (access control list). * * @return boolean true if there is also an ACL (access control list). + * @deprecated Get acl bit from the {@link org.apache.hadoop.fs.FileStatus} + * object. */ + @Deprecated public boolean getAclBit() { // File system subclasses that support the ACL bit would override this. return false; @@ -307,14 +313,20 @@ public class FsPermission implements Writable, Serializable, /** * Returns true if the file is encrypted or directory is in an encryption zone + * @deprecated Get encryption bit from the + * {@link org.apache.hadoop.fs.FileStatus} object. */ + @Deprecated public boolean getEncryptedBit() { return false; } /** * Returns true if the file or directory is erasure coded. + * @deprecated Get ec bit from the {@link org.apache.hadoop.fs.FileStatus} + * object. */ + @Deprecated public boolean getErasureCodedBit() { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java new file mode 100644 index 0000000..aa7d8f5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.protocolPB; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; + +import java.io.IOException; + +import static org.apache.hadoop.fs.FSProtos.*; + +/** + * Utility methods aiding conversion of fs data structures. + */ +public final class PBHelper { + + private PBHelper() { + // prevent construction + } + + public static FsPermission convert(FsPermissionProto proto) + throws IOException { + return new FsPermission((short)proto.getPerm()); + } + + public static FsPermissionProto convert(FsPermission p) throws IOException { + FsPermissionProto.Builder bld = FsPermissionProto.newBuilder(); + bld.setPerm(p.toShort()); + return bld.build(); + } + + public static FileStatus convert(FileStatusProto proto) throws IOException { + final Path path; + final long length; + final boolean isdir; + final short blockReplication; + final long blocksize; + final long mtime; + final long atime; + final String owner; + final String group; + final FsPermission permission; + final Path symlink; + switch (proto.getFileType()) { + case FT_DIR: + isdir = true; + symlink = null; + blocksize = 0; + length = 0; + blockReplication = 0; + break; + case FT_SYMLINK: + isdir = false; + symlink = new Path(proto.getSymlink()); + blocksize = 0; + length = 0; + blockReplication = 0; + break; + case FT_FILE: + isdir = false; + symlink = null; + blocksize = proto.getBlockSize(); + length = proto.getLength(); + int brep = proto.getBlockReplication(); + if ((brep & 0xffff0000) != 0) { + throw new IOException(String.format("Block replication 0x%08x " + + "doesn't fit in 16 bits.", brep)); + } + blockReplication = (short)brep; + break; + default: + throw new IllegalStateException("Unknown type: " + proto.getFileType()); + } + path = new Path(proto.getPath()); + mtime = proto.getModificationTime(); + atime = proto.getAccessTime(); + permission = convert(proto.getPermission()); + owner = proto.getOwner(); + group = proto.getGroup(); + int flags = proto.getFlags(); + return new FileStatus(length, isdir, blockReplication, blocksize, + mtime, atime, permission, owner, group, symlink, path, + (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0, + (flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0, + (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0); + } + + public static FileStatusProto convert(FileStatus stat) throws IOException { + FileStatusProto.Builder bld = FileStatusProto.newBuilder(); + bld.setPath(stat.getPath().toString()); + if (stat.isDirectory()) { + bld.setFileType(FileStatusProto.FileType.FT_DIR); + } else if (stat.isSymlink()) { + bld.setFileType(FileStatusProto.FileType.FT_SYMLINK) + .setSymlink(stat.getSymlink().toString()); + } else { + bld.setFileType(FileStatusProto.FileType.FT_FILE) + .setLength(stat.getLen()) + .setBlockReplication(stat.getReplication()) + .setBlockSize(stat.getBlockSize()); + } + bld.setAccessTime(stat.getAccessTime()) + .setModificationTime(stat.getModificationTime()) + .setOwner(stat.getOwner()) + .setGroup(stat.getGroup()) + .setPermission(convert(stat.getPermission())); + int flags = 0; + flags |= stat.hasAcl() ? FileStatusProto.Flags.HAS_ACL_VALUE : 0; + flags |= stat.isEncrypted() ? FileStatusProto.Flags.HAS_CRYPT_VALUE : 0; + flags |= stat.isErasureCoded() ? FileStatusProto.Flags.HAS_EC_VALUE : 0; + bld.setFlags(flags); + return bld.build(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java new file mode 100644 index 0000000..dbc9bce --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.protocolPB; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java index 4d66019..f008e24 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.io.erasurecode; +import java.io.Serializable; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -31,7 +32,10 @@ import org.apache.hadoop.classification.InterfaceStability; */ @InterfaceAudience.Public @InterfaceStability.Evolving -public final class ECSchema { +public final class ECSchema implements Serializable { + + private static final long serialVersionUID = 0x10953aa0; + public static final String NUM_DATA_UNITS_KEY = "numDataUnits"; public static final String NUM_PARITY_UNITS_KEY = "numParityUnits"; public static final String CODEC_NAME_KEY = "codec"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto new file mode 100644 index 0000000..6fe7980 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and stable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *stable* .proto interface. + */ + +option java_package = "org.apache.hadoop.fs"; +option java_outer_classname = "FSProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.fs; + +message FsPermissionProto { + required uint32 perm = 1; // UNIX-style mode bits +} + +/* + * FileStatus encoding. Field IDs match those from HdfsFileStatusProto, but + * cross-serialization is not an explicitly supported use case. Unlike HDFS, + * most fields are optional and do not define defaults. + */ +message FileStatusProto { + enum FileType { + FT_DIR = 1; + FT_FILE = 2; + FT_SYMLINK = 3; + } + enum Flags { + HAS_ACL = 0x01; // has ACLs + HAS_CRYPT = 0x02; // encrypted + HAS_EC = 0x04; // erasure coded + } + required FileType fileType = 1; + required string path = 2; + optional uint64 length = 3; + optional FsPermissionProto permission = 4; + optional string owner = 5; + optional string group = 6; + optional uint64 modification_time = 7; + optional uint64 access_time = 8; + optional string symlink = 9; + optional uint32 block_replication = 10; + optional uint64 block_size = 11; + // locations = 12 + // alias = 13 + // childrenNum = 14 + optional bytes encryption_data = 15; + // storagePolicy = 16 + optional bytes ec_data = 17; + optional uint32 flags = 18 [default = 0]; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java index 1962f49..61a688e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java @@ -36,6 +36,7 @@ import org.junit.Test; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java new file mode 100644 index 0000000..31cacf7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.protocolPB; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import static org.apache.hadoop.fs.FSProtos.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Verify PB serialization of FS data structures. + */ +public class TestFSSerialization { + + @Test + @SuppressWarnings("deprecation") + public void testWritableFlagSerialization() throws Exception { + final Path p = new Path("hdfs://yaks:4344/dingos/f"); + for (int i = 0; i < 0x8; ++i) { + final boolean acl = 0 != (i & 0x1); + final boolean crypt = 0 != (i & 0x2); + final boolean ec = 0 != (i & 0x4); + FileStatus stat = new FileStatus(1024L, false, 3, 1L << 31, + 12345678L, 87654321L, FsPermission.getFileDefault(), + "hadoop", "unqbbc", null, p, acl, crypt, ec); + DataOutputBuffer dob = new DataOutputBuffer(); + stat.write(dob); + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(dob.getData(), 0, dob.getLength()); + FileStatus fstat = new FileStatus(); + fstat.readFields(dib); + assertEquals(stat, fstat); + checkFields(stat, fstat); + } + } + + @Test + public void testUtilitySerialization() throws Exception { + final Path p = new Path("hdfs://yaks:4344/dingos/f"); + FileStatus stat = new FileStatus(1024L, false, 3, 1L << 31, + 12345678L, 87654321L, FsPermission.createImmutable((short)0111), + "hadoop", "unqbbc", null, p); + FileStatusProto fsp = PBHelper.convert(stat); + FileStatus stat2 = PBHelper.convert(fsp); + assertEquals(stat, stat2); + checkFields(stat, stat2); + } + + private static void checkFields(FileStatus expected, FileStatus actual) { + assertEquals(expected.getPath(), actual.getPath()); + assertEquals(expected.isDirectory(), actual.isDirectory()); + assertEquals(expected.getLen(), actual.getLen()); + assertEquals(expected.getPermission(), actual.getPermission()); + assertEquals(expected.getOwner(), actual.getOwner()); + assertEquals(expected.getGroup(), actual.getGroup()); + assertEquals(expected.getModificationTime(), actual.getModificationTime()); + assertEquals(expected.getAccessTime(), actual.getAccessTime()); + assertEquals(expected.getReplication(), actual.getReplication()); + assertEquals(expected.getBlockSize(), actual.getBlockSize()); + assertEquals(expected.hasAcl(), actual.hasAcl()); + assertEquals(expected.isEncrypted(), actual.isEncrypted()); + assertEquals(expected.isErasureCoded(), actual.isErasureCoded()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java index 368a2f2..7afc377 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.protocol; +import java.io.Serializable; + import com.google.common.base.Preconditions; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -29,11 +31,13 @@ import org.apache.hadoop.io.erasurecode.ECSchema; */ @InterfaceAudience.Public @InterfaceStability.Evolving -public final class ErasureCodingPolicy { +public final class ErasureCodingPolicy implements Serializable { + + private static final long serialVersionUID = 0x0079fe4e; + private String name; private final ECSchema schema; private final int cellSize; - private String name; private byte id; public ErasureCodingPolicy(String name, ECSchema schema, http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java index e0dd0d7..37d04e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java @@ -27,6 +27,11 @@ import org.apache.hadoop.fs.permission.FsPermission; * done for backwards compatibility in case any existing clients assume the * value of FsPermission is in a particular range. */ + +/** + * @deprecated ACLs, encryption, and erasure coding are managed on FileStatus. + */ +@Deprecated @InterfaceAudience.Private public class FsPermissionExtension extends FsPermission { private static final long serialVersionUID = 0x13c298a4; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index c386602..8438b01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.protocol; +import java.io.IOException; import java.net.URI; +import java.util.EnumSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -31,24 +33,15 @@ import org.apache.hadoop.hdfs.DFSUtilClient; */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class HdfsFileStatus { +public class HdfsFileStatus extends FileStatus { + + private static final long serialVersionUID = 0x126eb82a; // local name of the inode that's encoded in java UTF8 - private final byte[] path; - private final byte[] symlink; // symlink target encoded in java UTF8 or null - private final long length; - private final boolean isdir; - private final short block_replication; - private final long blocksize; - private final long modification_time; - private final long access_time; - private final FsPermission permission; - private final String owner; - private final String group; + private byte[] uPath; + private byte[] uSymlink; // symlink target encoded in java UTF8/null private final long fileId; - private final FileEncryptionInfo feInfo; - private final ErasureCodingPolicy ecPolicy; // Used by dir, not including dot and dotdot. Always zero for a regular file. @@ -58,11 +51,21 @@ public class HdfsFileStatus { public static final byte[] EMPTY_NAME = new byte[0]; /** + * Set of features potentially active on an instance. + */ + public enum Flags { + HAS_ACL, + HAS_CRYPT, + HAS_EC; + } + private final EnumSet<Flags> flags; + + /** * Constructor. - * @param length the number of bytes the file has - * @param isdir if the path is a directory + * @param length the number of bytes the file has + * @param isdir if the path is a directory * @param block_replication the replication factor - * @param blocksize the block size + * @param blocksize the block size * @param modification_time modification time * @param access_time access time * @param permission permission @@ -77,25 +80,18 @@ public class HdfsFileStatus { * @param ecPolicy the erasure coding policy */ public HdfsFileStatus(long length, boolean isdir, int block_replication, - long blocksize, long modification_time, long access_time, - FsPermission permission, String owner, String group, byte[] symlink, - byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, - byte storagePolicy, ErasureCodingPolicy ecPolicy) { - this.length = length; - this.isdir = isdir; - this.block_replication = (short) block_replication; - this.blocksize = blocksize; - this.modification_time = modification_time; - this.access_time = access_time; - this.permission = (permission == null) ? - ((isdir || symlink!=null) ? - FsPermission.getDefault() : - FsPermission.getFileDefault()) : - permission; - this.owner = (owner == null) ? "" : owner; - this.group = (group == null) ? "" : group; - this.symlink = symlink; - this.path = path; + long blocksize, long modification_time, + long access_time, FsPermission permission, + EnumSet<Flags> flags, String owner, String group, + byte[] symlink, byte[] path, long fileId, + int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy, ErasureCodingPolicy ecPolicy) { + super(length, isdir, block_replication, blocksize, modification_time, + access_time, convert(isdir, symlink != null, permission, flags), + owner, group, null, null); + this.flags = flags; + this.uSymlink = symlink; + this.uPath = path; this.fileId = fileId; this.childrenNum = childrenNum; this.feInfo = feInfo; @@ -104,83 +100,48 @@ public class HdfsFileStatus { } /** - * Get the length of this file, in bytes. - * @return the length of this file, in bytes. + * Set redundant flags for compatibility with existing applications. */ - public final long getLen() { - return length; - } - - /** - * Is this a directory? - * @return true if this is a directory - */ - public final boolean isDir() { - return isdir; + protected static FsPermission convert(boolean isdir, boolean symlink, + FsPermission p, EnumSet<Flags> f) { + if (p instanceof FsPermissionExtension) { + // verify flags are set consistently + assert p.getAclBit() == f.contains(HdfsFileStatus.Flags.HAS_ACL); + assert p.getEncryptedBit() == f.contains(HdfsFileStatus.Flags.HAS_CRYPT); + assert p.getErasureCodedBit() == f.contains(HdfsFileStatus.Flags.HAS_EC); + return p; + } + if (null == p) { + if (isdir) { + p = FsPermission.getDirDefault(); + } else if (symlink) { + p = FsPermission.getDefault(); + } else { + p = FsPermission.getFileDefault(); + } + } + return new FsPermissionExtension(p, f.contains(Flags.HAS_ACL), + f.contains(Flags.HAS_CRYPT), f.contains(Flags.HAS_EC)); } - /** - * Is this a symbolic link? - * @return true if this is a symbolic link - */ + @Override public boolean isSymlink() { - return symlink != null; + return uSymlink != null; } - /** - * Get the block size of the file. - * @return the number of bytes - */ - public final long getBlockSize() { - return blocksize; + @Override + public boolean hasAcl() { + return flags.contains(Flags.HAS_ACL); } - /** - * Get the replication factor of a file. - * @return the replication factor of a file. - */ - public final short getReplication() { - return block_replication; + @Override + public boolean isEncrypted() { + return flags.contains(Flags.HAS_CRYPT); } - /** - * Get the modification time of the file. - * @return the modification time of file in milliseconds since January 1, 1970 UTC. - */ - public final long getModificationTime() { - return modification_time; - } - - /** - * Get the access time of the file. - * @return the access time of file in milliseconds since January 1, 1970 UTC. - */ - public final long getAccessTime() { - return access_time; - } - - /** - * Get FsPermission associated with the file. - * @return permission - */ - public final FsPermission getPermission() { - return permission; - } - - /** - * Get the owner of the file. - * @return owner of the file - */ - public final String getOwner() { - return owner; - } - - /** - * Get the group associated with the file. - * @return group for the file. - */ - public final String getGroup() { - return group; + @Override + public boolean isErasureCoded() { + return flags.contains(Flags.HAS_EC); } /** @@ -188,7 +149,7 @@ public class HdfsFileStatus { * @return true if the name is empty */ public final boolean isEmptyLocalName() { - return path.length == 0; + return uPath.length == 0; } /** @@ -196,7 +157,7 @@ public class HdfsFileStatus { * @return the local name in string */ public final String getLocalName() { - return DFSUtilClient.bytes2String(path); + return DFSUtilClient.bytes2String(uPath); } /** @@ -204,7 +165,7 @@ public class HdfsFileStatus { * @return the local name in java UTF8 */ public final byte[] getLocalNameInBytes() { - return path; + return uPath; } /** @@ -238,16 +199,24 @@ public class HdfsFileStatus { return new Path(parent, getLocalName()); } - /** - * Get the string representation of the symlink. - * @return the symlink as a string. - */ - public final String getSymlink() { - return DFSUtilClient.bytes2String(symlink); + @Override + public Path getSymlink() throws IOException { + if (isSymlink()) { + return new Path(DFSUtilClient.bytes2String(uSymlink)); + } + throw new IOException("Path " + getPath() + " is not a symbolic link"); + } + + @Override + public void setSymlink(Path sym) { + uSymlink = DFSUtilClient.string2Bytes(sym.toString()); } + /** + * Opaque referant for the symlink, to be resolved at the client. + */ public final byte[] getSymlinkInBytes() { - return symlink; + return uSymlink; } public final long getFileId() { @@ -275,13 +244,30 @@ public class HdfsFileStatus { return storagePolicy; } - public final FileStatus makeQualified(URI defaultUri, Path path) { - return new FileStatus(getLen(), isDir(), getReplication(), - getBlockSize(), getModificationTime(), - getAccessTime(), - getPermission(), getOwner(), getGroup(), - isSymlink() ? new Path(getSymlink()) : null, - (getFullPath(path)).makeQualified( - defaultUri, null)); // fully-qualify path + @Override + public boolean equals(Object o) { + // satisfy findbugs + return super.equals(o); + } + + @Override + public int hashCode() { + // satisfy findbugs + return super.hashCode(); + } + + /** + * Resolve the short name of the Path given the URI, parent provided. This + * FileStatus reference will not contain a valid Path until it is resolved + * by this method. + * @param defaultUri FileSystem to fully qualify HDFS path. + * @param parent Parent path of this element. + * @return Reference to this instance. + */ + public final FileStatus makeQualified(URI defaultUri, Path parent) { + // fully-qualify path + setPath(getFullPath(parent).makeQualified(defaultUri, null)); + return this; // API compatibility } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 0fd2039..b82a860 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.protocol; import java.net.URI; +import java.util.EnumSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -34,7 +35,14 @@ import org.apache.hadoop.hdfs.DFSUtilClient; @InterfaceAudience.Private @InterfaceStability.Evolving public class HdfsLocatedFileStatus extends HdfsFileStatus { - private final LocatedBlocks locations; + + private static final long serialVersionUID = 0x23c73328; + + /** + * Left transient, because {@link #makeQualifiedLocated(URI,Path)} + * is the user-facing type. + */ + private transient LocatedBlocks locations; /** * Constructor @@ -56,12 +64,12 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { */ public HdfsLocatedFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, - long access_time, FsPermission permission, String owner, String group, - byte[] symlink, byte[] path, long fileId, LocatedBlocks locations, - int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy, - ErasureCodingPolicy ecPolicy) { + long access_time, FsPermission permission, EnumSet<Flags> flags, + String owner, String group, byte[] symlink, byte[] path, long fileId, + LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy, ErasureCodingPolicy ecPolicy) { super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, owner, group, symlink, path, fileId, + access_time, permission, flags, owner, group, symlink, path, fileId, childrenNum, feInfo, storagePolicy, ecPolicy); this.locations = locations; } @@ -72,13 +80,21 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { public final LocatedFileStatus makeQualifiedLocated(URI defaultUri, Path path) { - return new LocatedFileStatus(getLen(), isDir(), getReplication(), - getBlockSize(), getModificationTime(), - getAccessTime(), - getPermission(), getOwner(), getGroup(), - isSymlink() ? new Path(getSymlink()) : null, - (getFullPath(path)).makeQualified( - defaultUri, null), // fully-qualify path + makeQualified(defaultUri, path); + return new LocatedFileStatus(this, DFSUtilClient.locatedBlocks2Locations(getBlockLocations())); } + + @Override + public boolean equals(Object o) { + // satisfy findbugs + return super.equals(o); + } + + @Override + public int hashCode() { + // satisfy findbugs + return super.hashCode(); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java index 583d027..61e5316 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java @@ -21,6 +21,7 @@ import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.Comparator; import java.util.Date; +import java.util.EnumSet; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -57,11 +58,12 @@ public class SnapshottableDirectoryStatus { private final byte[] parentFullPath; public SnapshottableDirectoryStatus(long modification_time, long access_time, - FsPermission permission, String owner, String group, byte[] localName, - long inodeId, int childrenNum, - int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { + FsPermission permission, EnumSet<HdfsFileStatus.Flags> flags, + String owner, String group, byte[] localName, long inodeId, + int childrenNum, int snapshotNumber, int snapshotQuota, + byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, - access_time, permission, owner, group, null, localName, inodeId, + access_time, permission, flags, owner, group, null, localName, inodeId, childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null); this.snapshotNumber = snapshotNumber; http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index feb3061..a0d2c5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntrySco import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto; +import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockFlagProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto; @@ -149,7 +150,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.Sto import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType; @@ -1142,7 +1142,7 @@ public class PBHelperClient { } public static FsPermission convert(FsPermissionProto p) { - return new FsPermissionExtension((short)p.getPerm()); + return new FsPermission((short)p.getPerm()); } private static Event.CreateEvent.INodeType createTypeConvert( @@ -1501,10 +1501,14 @@ public class PBHelperClient { return null; } final HdfsFileStatusProto status = sdirStatusProto.getDirStatus(); + EnumSet<HdfsFileStatus.Flags> flags = status.hasFlags() + ? convertFlags(status.getFlags()) + : convertFlags(status.getPermission()); return new SnapshottableDirectoryStatus( status.getModificationTime(), status.getAccessTime(), convert(status.getPermission()), + flags, status.getOwner(), status.getGroup(), status.getPath().toByteArray(), @@ -1546,17 +1550,23 @@ public class PBHelperClient { } public static FsPermissionProto convert(FsPermission p) { - return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build(); + return FsPermissionProto.newBuilder().setPerm(p.toShort()).build(); } public static HdfsFileStatus convert(HdfsFileStatusProto fs) { - if (fs == null) + if (fs == null) { return null; + } + EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags() + ? convertFlags(fs.getFlags()) + : convertFlags(fs.getPermission()); return new HdfsLocatedFileStatus( fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), fs.getBlockReplication(), fs.getBlocksize(), fs.getModificationTime(), fs.getAccessTime(), - convert(fs.getPermission()), fs.getOwner(), fs.getGroup(), + convert(fs.getPermission()), + flags, + fs.getOwner(), fs.getGroup(), fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), @@ -1569,6 +1579,47 @@ public class PBHelperClient { fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null); } + private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) { + EnumSet<HdfsFileStatus.Flags> f = + EnumSet.noneOf(HdfsFileStatus.Flags.class); + for (HdfsFileStatusProto.Flags pbf : HdfsFileStatusProto.Flags.values()) { + if ((pbf.getNumber() & flags) != 0) { + switch (pbf) { + case HAS_ACL: + f.add(HdfsFileStatus.Flags.HAS_ACL); + break; + case HAS_CRYPT: + f.add(HdfsFileStatus.Flags.HAS_CRYPT); + break; + case HAS_EC: + f.add(HdfsFileStatus.Flags.HAS_EC); + break; + default: + // ignore unknown + break; + } + } + } + return f; + } + + private static EnumSet<HdfsFileStatus.Flags> convertFlags( + FsPermissionProto pbp) { + EnumSet<HdfsFileStatus.Flags> f = + EnumSet.noneOf(HdfsFileStatus.Flags.class); + FsPermission p = new FsPermissionExtension((short)pbp.getPerm()); + if (p.getAclBit()) { + f.add(HdfsFileStatus.Flags.HAS_ACL); + } + if (p.getEncryptedBit()) { + f.add(HdfsFileStatus.Flags.HAS_CRYPT); + } + if (p.getErasureCodedBit()) { + f.add(HdfsFileStatus.Flags.HAS_EC); + } + return f; + } + public static CorruptFileBlocks convert(CorruptFileBlocksProto c) { if (c == null) return null; @@ -2082,6 +2133,10 @@ public class PBHelperClient { builder.setEcPolicy(convertErasureCodingPolicy( fs.getErasureCodingPolicy())); } + int flags = fs.hasAcl() ? HdfsFileStatusProto.Flags.HAS_ACL_VALUE : 0; + flags |= fs.isEncrypted() ? HdfsFileStatusProto.Flags.HAS_CRYPT_VALUE : 0; + flags |= fs.isErasureCoded() ? HdfsFileStatusProto.Flags.HAS_EC_VALUE : 0; + builder.setFlags(flags); return builder.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 5e9396e..7ec5fe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -61,6 +60,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -97,17 +97,8 @@ class JsonUtilClient { } /** Convert a string to a FsPermission object. */ - static FsPermission toFsPermission( - final String s, Boolean aclBit, Boolean encBit, Boolean erasureBit) { - FsPermission perm = new FsPermission(Short.parseShort(s, 8)); - final boolean aBit = (aclBit != null) ? aclBit : false; - final boolean eBit = (encBit != null) ? encBit : false; - final boolean ecBit = (erasureBit != null) ? erasureBit : false; - if (aBit || eBit || ecBit) { - return new FsPermissionExtension(perm, aBit, eBit, ecBit); - } else { - return perm; - } + static FsPermission toFsPermission(final String s) { + return null == s ? null : new FsPermission(Short.parseShort(s, 8)); } /** Convert a Json map to a HdfsFileStatus object. */ @@ -128,10 +119,23 @@ class JsonUtilClient { final long len = ((Number) m.get("length")).longValue(); final String owner = (String) m.get("owner"); final String group = (String) m.get("group"); - final FsPermission permission = toFsPermission((String) m.get("permission"), - (Boolean) m.get("aclBit"), - (Boolean) m.get("encBit"), - (Boolean) m.get("ecBit")); + final FsPermission permission = toFsPermission((String)m.get("permission")); + + Boolean aclBit = (Boolean) m.get("aclBit"); + Boolean encBit = (Boolean) m.get("encBit"); + Boolean erasureBit = (Boolean) m.get("ecBit"); + EnumSet<HdfsFileStatus.Flags> f = + EnumSet.noneOf(HdfsFileStatus.Flags.class); + if (aclBit != null && aclBit) { + f.add(HdfsFileStatus.Flags.HAS_ACL); + } + if (encBit != null && encBit) { + f.add(HdfsFileStatus.Flags.HAS_CRYPT); + } + if (erasureBit != null && erasureBit) { + f.add(HdfsFileStatus.Flags.HAS_EC); + } + final long aTime = ((Number) m.get("accessTime")).longValue(); final long mTime = ((Number) m.get("modificationTime")).longValue(); final long blockSize = ((Number) m.get("blockSize")).longValue(); @@ -143,11 +147,11 @@ class JsonUtilClient { final byte storagePolicy = m.containsKey("storagePolicy") ? (byte) ((Number) m.get("storagePolicy")).longValue() : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; - return new HdfsFileStatus(len, type == WebHdfsConstants.PathType.DIRECTORY, - replication, blockSize, mTime, aTime, permission, owner, group, - symlink, DFSUtilClient.string2Bytes(localName), - fileId, childrenNum, null, - storagePolicy, null); + return new HdfsFileStatus(len, + type == WebHdfsConstants.PathType.DIRECTORY, replication, blockSize, + mTime, aTime, permission, f, owner, group, symlink, + DFSUtilClient.string2Bytes(localName), fileId, childrenNum, + null, storagePolicy, null); } static HdfsFileStatus[] toHdfsFileStatusArray(final Map<?, ?> json) { @@ -465,9 +469,7 @@ class JsonUtilClient { aclStatusBuilder.stickyBit((Boolean) m.get("stickyBit")); String permString = (String) m.get("permission"); if (permString != null) { - final FsPermission permission = toFsPermission(permString, - (Boolean) m.get("aclBit"), (Boolean) m.get("encBit"), - (Boolean) m.get("ecBit")); + final FsPermission permission = toFsPermission(permString); aclStatusBuilder.setPermission(permission); } final List<?> entries = (List<?>) m.get("entries"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java index 50da899..f690dd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java @@ -32,7 +32,13 @@ public class WebHdfsConstants { FILE, DIRECTORY, SYMLINK; static PathType valueOf(HdfsFileStatus status) { - return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE; + if (status.isDirectory()) { + return DIRECTORY; + } + if (status.isSymlink()) { + return SYMLINK; + } + return FILE; } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 3861cba..1159e50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -1016,15 +1016,7 @@ public class WebHdfsFileSystem extends FileSystem public FileStatus getFileStatus(Path f) throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_FILE_STATUS); - return makeQualified(getHdfsFileStatus(f), f); - } - - private FileStatus makeQualified(HdfsFileStatus f, Path parent) { - return new FileStatus(f.getLen(), f.isDir(), f.getReplication(), - f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), - f.getPermission(), f.getOwner(), f.getGroup(), - f.isSymlink() ? new Path(f.getSymlink()) : null, - f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory())); + return getHdfsFileStatus(f).makeQualified(getUri(), f); } @Override @@ -1507,6 +1499,7 @@ public class WebHdfsFileSystem extends FileSystem statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + final URI fsUri = getUri(); final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS; return new FsPathResponseRunner<FileStatus[]>(op, f) { @Override @@ -1515,7 +1508,7 @@ public class WebHdfsFileSystem extends FileSystem JsonUtilClient.toHdfsFileStatusArray(json); final FileStatus[] statuses = new FileStatus[hdfsStatuses.length]; for (int i = 0; i < hdfsStatuses.length; i++) { - statuses[i] = makeQualified(hdfsStatuses[i], f); + statuses[i] = hdfsStatuses[i].makeQualified(fsUri, f); } return statuses; @@ -1541,10 +1534,11 @@ public class WebHdfsFileSystem extends FileSystem } }.run(); // Qualify the returned FileStatus array + final URI fsUri = getUri(); final HdfsFileStatus[] statuses = listing.getPartialListing(); FileStatus[] qualified = new FileStatus[statuses.length]; for (int i = 0; i < statuses.length; i++) { - qualified[i] = makeQualified(statuses[i], f); + qualified[i] = statuses[i].makeQualified(fsUri, f); } return new DirectoryEntries(qualified, listing.getLastName(), listing.hasMore()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto index bb7fdb0..c2529c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto @@ -21,7 +21,12 @@ option java_outer_classname = "AclProtos"; option java_generate_equals_and_hash = true; package hadoop.hdfs; -import "hdfs.proto"; +/** + * File or Directory permision - same spec as posix + */ +message FsPermissionProto { + required uint32 perm = 1; // Actually a short - only 16bits used +} message AclEntryProto { enum AclEntryScopeProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index b306fcf..7109980 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -32,6 +32,7 @@ option java_generate_equals_and_hash = true; package hadoop.hdfs; import "Security.proto"; +import "acl.proto"; /** * Extended block idenfies a block @@ -197,13 +198,6 @@ message CorruptFileBlocksProto { } /** - * File or Directory permision - same spec as posix - */ -message FsPermissionProto { - required uint32 perm = 1; // Actually a short - only 16bits used -} - -/** * Types of recognized storage media. */ enum StorageTypeProto { @@ -388,6 +382,11 @@ message HdfsFileStatusProto { IS_FILE = 2; IS_SYMLINK = 3; } + enum Flags { + HAS_ACL = 0x01; // has ACLs + HAS_CRYPT = 0x02; // encrypted + HAS_EC = 0x04; // erasure coded + } required FileType fileType = 1; required bytes path = 2; // local name of inode encoded java UTF8 required uint64 length = 3; @@ -415,6 +414,9 @@ message HdfsFileStatusProto { // Optional field for erasure coding optional ErasureCodingPolicyProto ecPolicy = 17; + + // Set of flags + optional uint32 flags = 18 [default = 0]; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 1ab890f..d139100 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -1047,18 +1047,7 @@ public class HttpFSFileSystem extends FileSystem /** Convert a string to a FsPermission object. */ static FsPermission toFsPermission(JSONObject json) { final String s = (String) json.get(PERMISSION_JSON); - final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON); - final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON); - final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON); - FsPermission perm = new FsPermission(Short.parseShort(s, 8)); - final boolean aBit = (aclBit != null) ? aclBit : false; - final boolean eBit = (encBit != null) ? encBit : false; - final boolean ecBit = (erasureBit != null) ? erasureBit : false; - if (aBit || eBit || ecBit) { - return new FsPermissionExtension(perm, aBit, eBit, ecBit); - } else { - return perm; - } + return new FsPermission(Short.parseShort(s, 8)); } private FileStatus createFileStatus(Path parent, JSONObject json) { @@ -1073,23 +1062,23 @@ public class HttpFSFileSystem extends FileSystem long mTime = (Long) json.get(MODIFICATION_TIME_JSON); long blockSize = (Long) json.get(BLOCK_SIZE_JSON); short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); - FileStatus fileStatus = null; - - switch (type) { - case FILE: - case DIRECTORY: - fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), - replication, blockSize, mTime, aTime, - permission, owner, group, path); - break; - case SYMLINK: - Path symLink = null; - fileStatus = new FileStatus(len, false, - replication, blockSize, mTime, aTime, - permission, owner, group, symLink, - path); + + final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON); + final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON); + final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON); + final boolean aBit = (aclBit != null) ? aclBit : false; + final boolean eBit = (encBit != null) ? encBit : false; + final boolean ecBit = (erasureBit != null) ? erasureBit : false; + if (aBit || eBit || ecBit) { + // include this for compatibility with 2.x + FsPermissionExtension deprecatedPerm = + new FsPermissionExtension(permission, aBit, eBit, ecBit); + return new FileStatus(len, FILE_TYPE.DIRECTORY == type, + replication, blockSize, mTime, aTime, deprecatedPerm, owner, group, + null, path, aBit, eBit, ecBit); } - return fileStatus; + return new FileStatus(len, FILE_TYPE.DIRECTORY == type, + replication, blockSize, mTime, aTime, permission, owner, group, path); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index e23093e..ca11c66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -852,10 +852,11 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { } } - private static void assertSameAclBit(FileSystem expected, FileSystem actual, + private static void assertSameAcls(FileSystem expected, FileSystem actual, Path path) throws IOException { FileStatus expectedFileStatus = expected.getFileStatus(path); FileStatus actualFileStatus = actual.getFileStatus(path); + assertEquals(actualFileStatus.hasAcl(), expectedFileStatus.hasAcl()); assertEquals(actualFileStatus.getPermission().getAclBit(), expectedFileStatus.getPermission().getAclBit()); } @@ -888,31 +889,31 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { AclStatus proxyAclStat = proxyFs.getAclStatus(path); AclStatus httpfsAclStat = httpfs.getAclStatus(path); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, path); + assertSameAcls(httpfs, proxyFs, path); httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet,true)); proxyAclStat = proxyFs.getAclStatus(path); httpfsAclStat = httpfs.getAclStatus(path); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, path); + assertSameAcls(httpfs, proxyFs, path); httpfs.modifyAclEntries(path, AclEntry.parseAclSpec(aclUser2, true)); proxyAclStat = proxyFs.getAclStatus(path); httpfsAclStat = httpfs.getAclStatus(path); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, path); + assertSameAcls(httpfs, proxyFs, path); httpfs.removeAclEntries(path, AclEntry.parseAclSpec(rmAclUser1, false)); proxyAclStat = proxyFs.getAclStatus(path); httpfsAclStat = httpfs.getAclStatus(path); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, path); + assertSameAcls(httpfs, proxyFs, path); httpfs.removeAcl(path); proxyAclStat = proxyFs.getAclStatus(path); httpfsAclStat = httpfs.getAclStatus(path); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, path); + assertSameAcls(httpfs, proxyFs, path); } /** @@ -935,21 +936,21 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { AclStatus proxyAclStat = proxyFs.getAclStatus(dir); AclStatus httpfsAclStat = httpfs.getAclStatus(dir); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, dir); + assertSameAcls(httpfs, proxyFs, dir); /* Set a default ACL on the directory */ httpfs.setAcl(dir, (AclEntry.parseAclSpec(defUser1,true))); proxyAclStat = proxyFs.getAclStatus(dir); httpfsAclStat = httpfs.getAclStatus(dir); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, dir); + assertSameAcls(httpfs, proxyFs, dir); /* Remove the default ACL */ httpfs.removeDefaultAcl(dir); proxyAclStat = proxyFs.getAclStatus(dir); httpfsAclStat = httpfs.getAclStatus(dir); assertSameAcls(httpfsAclStat, proxyAclStat); - assertSameAclBit(httpfs, proxyFs, dir); + assertSameAcls(httpfs, proxyFs, dir); } private void testEncryption() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml index be54efb..2a7824a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml @@ -252,4 +252,16 @@ <Class name="org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture" /> <Bug pattern="NS_DANGEROUS_NON_SHORT_CIRCUIT" /> </Match> + <Match> + <Class name="org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil$1" /> + <Method name="visitFile" /> + <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" /> + </Match> + <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not. + Defensible compatibility choices over time create odd corners. --> + <Match> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> + <Field name="locations" /> + <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" /> + </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java index 3ddfc85..676e827 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto; +import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java new file mode 100644 index 0000000..6233024 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocolPB; \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
