Revert "HDFS-12681. Fold HdfsLocatedFileStatus into HdfsFileStatus."
This reverts commit b85603e3f85e85da406241b991f3a9974384c3aa. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675e9a8f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675e9a8f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675e9a8f Branch: refs/heads/YARN-6592 Commit: 675e9a8f57570771a0219d95940681b067d36b94 Parents: b1941b2 Author: Chris Douglas <cdoug...@apache.org> Authored: Wed Nov 15 19:17:46 2017 -0800 Committer: Chris Douglas <cdoug...@apache.org> Committed: Wed Nov 15 19:20:37 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/LocatedFileStatus.java | 11 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +- .../hadoop/hdfs/DistributedFileSystem.java | 4 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 80 ++++---------- .../hdfs/protocol/HdfsLocatedFileStatus.java | 110 +++++++++++++++++++ .../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +++++------ .../dev-support/findbugsExcludeFile.xml | 7 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 6 +- .../server/namenode/FSDirStatAndListingOp.java | 61 +++++----- .../hadoop/hdfs/TestBlockStoragePolicy.java | 10 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 2 +- .../hdfs/server/mover/TestStorageMover.java | 8 +- 12 files changed, 217 insertions(+), 151 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java index 9cc81d3..29e1998 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java @@ -135,16 +135,7 @@ public class LocatedFileStatus extends FileStatus { public BlockLocation[] getBlockLocations() { return locations; } - - /** - * Hook for subclasses to lazily set block locations. The {@link #locations} - * field should be null before this is called. - * @param locations Block locations for this instance. - */ - protected void setBlockLocations(BlockLocation[] locations) { - this.locations = locations; - } - + /** * Compare this FileStatus to another FileStatus * @param o the FileStatus to be compared. http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index f306d06..0138195 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; @@ -187,7 +188,8 @@ public class Hdfs extends AbstractFileSystem { @Override public LocatedFileStatus next() throws IOException { - return getNext().makeQualifiedLocated(getUri(), p); + return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated( + getUri(), p); } }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index eef83d7..9db12e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -85,6 +85,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsPathHandle; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; @@ -1211,7 +1212,8 @@ public class DistributedFileSystem extends FileSystem { T next; HdfsFileStatus fileStat = thisListing.getPartialListing()[i++]; if (needLocation) { - next = (T)fileStat.makeQualifiedLocated(getUri(), p); + next = (T)((HdfsLocatedFileStatus)fileStat) + .makeQualifiedLocated(getUri(), p); } else { next = (T)fileStat.makeQualified(getUri(), p); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index fce2c44..0499f2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -26,17 +26,15 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtilClient; -/** - * HDFS metadata for an entity in the filesystem. +/** Interface that represents the over the wire information for a file. */ @InterfaceAudience.Private @InterfaceStability.Evolving -public final class HdfsFileStatus extends LocatedFileStatus { +public class HdfsFileStatus extends FileStatus { private static final long serialVersionUID = 0x126eb82a; @@ -51,12 +49,11 @@ public final class HdfsFileStatus extends LocatedFileStatus { private final int childrenNum; private final byte storagePolicy; - // BlockLocations[] is the user-facing type - private transient LocatedBlocks hdfsloc; - public static final byte[] EMPTY_NAME = new byte[0]; - /** Set of features potentially active on an instance. */ + /** + * Set of features potentially active on an instance. + */ public enum Flags { HAS_ACL, HAS_CRYPT, @@ -84,19 +81,18 @@ public final class HdfsFileStatus extends LocatedFileStatus { * @param storagePolicy ID which specifies storage policy * @param ecPolicy the erasure coding policy */ - private HdfsFileStatus(long length, boolean isdir, int replication, + protected HdfsFileStatus(long length, boolean isdir, int replication, long blocksize, long mtime, long atime, FsPermission permission, EnumSet<Flags> flags, String owner, String group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, - byte storagePolicy, ErasureCodingPolicy ecPolicy, - LocatedBlocks hdfsloc) { + byte storagePolicy, ErasureCodingPolicy ecPolicy) { super(length, isdir, replication, blocksize, mtime, atime, convert(isdir, symlink != null, permission, flags), owner, group, null, null, flags.contains(Flags.HAS_ACL), flags.contains(Flags.HAS_CRYPT), - flags.contains(Flags.HAS_EC), null); + flags.contains(Flags.HAS_EC)); this.flags = flags; this.uSymlink = symlink; this.uPath = path; @@ -105,7 +101,6 @@ public final class HdfsFileStatus extends LocatedFileStatus { this.feInfo = feInfo; this.storagePolicy = storagePolicy; this.ecPolicy = ecPolicy; - this.hdfsloc = hdfsloc; } /** @@ -157,7 +152,7 @@ public final class HdfsFileStatus extends LocatedFileStatus { * Check if the local name is empty. * @return true if the name is empty */ - public boolean isEmptyLocalName() { + public final boolean isEmptyLocalName() { return uPath.length == 0; } @@ -165,7 +160,7 @@ public final class HdfsFileStatus extends LocatedFileStatus { * Get the string representation of the local name. * @return the local name in string */ - public String getLocalName() { + public final String getLocalName() { return DFSUtilClient.bytes2String(uPath); } @@ -173,7 +168,7 @@ public final class HdfsFileStatus extends LocatedFileStatus { * Get the Java UTF8 representation of the local name. * @return the local name in java UTF8 */ - public byte[] getLocalNameInBytes() { + public final byte[] getLocalNameInBytes() { return uPath; } @@ -182,7 +177,7 @@ public final class HdfsFileStatus extends LocatedFileStatus { * @param parent the parent path * @return the full path in string */ - public String getFullName(String parent) { + public final String getFullName(final String parent) { if (isEmptyLocalName()) { return parent; } @@ -200,7 +195,7 @@ public final class HdfsFileStatus extends LocatedFileStatus { * @param parent the parent path * @return the full path */ - public Path getFullPath(Path parent) { + public final Path getFullPath(final Path parent) { if (isEmptyLocalName()) { return parent; } @@ -224,15 +219,15 @@ public final class HdfsFileStatus extends LocatedFileStatus { /** * Opaque referant for the symlink, to be resolved at the client. */ - public byte[] getSymlinkInBytes() { + public final byte[] getSymlinkInBytes() { return uSymlink; } - public long getFileId() { + public final long getFileId() { return fileId; } - public FileEncryptionInfo getFileEncryptionInfo() { + public final FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } @@ -244,12 +239,12 @@ public final class HdfsFileStatus extends LocatedFileStatus { return ecPolicy; } - public int getChildrenNum() { + public final int getChildrenNum() { return childrenNum; } /** @return the storage policy id */ - public byte getStoragePolicy() { + public final byte getStoragePolicy() { return storagePolicy; } @@ -262,10 +257,6 @@ public final class HdfsFileStatus extends LocatedFileStatus { return flags.contains(Flags.SNAPSHOT_ENABLED); } - public LocatedBlocks getLocatedBlocks() { - return hdfsloc; - } - @Override public boolean equals(Object o) { // satisfy findbugs @@ -286,30 +277,11 @@ public final class HdfsFileStatus extends LocatedFileStatus { * @param parent Parent path of this element. * @return Reference to this instance. */ - public FileStatus makeQualified(URI defaultUri, Path parent) { + public final FileStatus makeQualified(URI defaultUri, Path parent) { // fully-qualify path setPath(getFullPath(parent).makeQualified(defaultUri, null)); return this; // API compatibility - } - /** - * This function is used to transform the underlying HDFS LocatedBlocks to - * BlockLocations. This method must be invoked before - * {@link #getBlockLocations()}. - * - * The returned BlockLocation will have different formats for replicated - * and erasure coded file. - * Please refer to - * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations - * (FileStatus, long, long)} - * for examples. - */ - public LocatedFileStatus makeQualifiedLocated(URI defaultUri, - Path path) { - makeQualified(defaultUri, path); - setBlockLocations( - DFSUtilClient.locatedBlocks2Locations(getLocatedBlocks())); - return this; } /** @@ -339,7 +311,6 @@ public final class HdfsFileStatus extends LocatedFileStatus { private byte storagePolicy = HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; private ErasureCodingPolicy ecPolicy = null; - private LocatedBlocks locations = null; /** * Set the length of the entity (default = 0). @@ -519,23 +490,12 @@ public final class HdfsFileStatus extends LocatedFileStatus { } /** - * Set the block locations for this entity (default = null). - * @param locations HDFS locations - * (see {@link #makeQualifiedLocated(URI, Path)}) - * @return This Builder instance - */ - public Builder locations(LocatedBlocks locations) { - this.locations = locations; - return this; - } - - /** * @return An {@link HdfsFileStatus} instance from these parameters. */ public HdfsFileStatus build() { return new HdfsFileStatus(length, isdir, replication, blocksize, mtime, atime, permission, flags, owner, group, symlink, path, fileId, - childrenNum, feInfo, storagePolicy, ecPolicy, locations); + childrenNum, feInfo, storagePolicy, ecPolicy); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java new file mode 100644 index 0000000..193aae2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.net.URI; +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSUtilClient; + +/** + * Interface that represents the over the wire information + * including block locations for a file. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class HdfsLocatedFileStatus extends HdfsFileStatus { + + private static final long serialVersionUID = 0x23c73328; + + /** + * Left transient, because {@link #makeQualifiedLocated(URI,Path)} + * is the user-facing type. + */ + private transient LocatedBlocks locations; + + /** + * Constructor + * + * @param length size + * @param isdir if this is directory + * @param block_replication the file's replication factor + * @param blocksize the file's block size + * @param modification_time most recent modification time + * @param access_time most recent access time + * @param permission permission + * @param owner owner + * @param group group + * @param symlink symbolic link + * @param path local path name in java UTF8 format + * @param fileId the file id + * @param locations block locations + * @param feInfo file encryption info + */ + public HdfsLocatedFileStatus(long length, boolean isdir, + int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, EnumSet<Flags> flags, + String owner, String group, byte[] symlink, byte[] path, long fileId, + LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy, ErasureCodingPolicy ecPolicy) { + super(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, flags, owner, group, symlink, path, fileId, + childrenNum, feInfo, storagePolicy, ecPolicy); + this.locations = locations; + } + + public LocatedBlocks getBlockLocations() { + return locations; + } + + /** + * This function is used to transform the underlying HDFS LocatedBlocks to + * BlockLocations. + * + * The returned BlockLocation will have different formats for replicated + * and erasure coded file. + * Please refer to + * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations + * (FileStatus, long, long)} + * for examples. + */ + public final LocatedFileStatus makeQualifiedLocated(URI defaultUri, + Path path) { + makeQualified(defaultUri, path); + return new LocatedFileStatus(this, + DFSUtilClient.locatedBlocks2Locations(getBlockLocations())); + } + + @Override + public boolean equals(Object o) { + // satisfy findbugs + return super.equals(o); + } + + @Override + public int hashCode() { + // satisfy findbugs + return super.hashCode(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 54fab6d..ae3921c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -90,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsPathHandle; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -1584,36 +1585,23 @@ public class PBHelperClient { EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags() ? convertFlags(fs.getFlags()) : convertFlags(fs.getPermission()); - return new HdfsFileStatus.Builder() - .length(fs.getLength()) - .isdir(fs.getFileType().equals(FileType.IS_DIR)) - .replication(fs.getBlockReplication()) - .blocksize(fs.getBlocksize()) - .mtime(fs.getModificationTime()) - .atime(fs.getAccessTime()) - .perm(convert(fs.getPermission())) - .flags(flags) - .owner(fs.getOwner()) - .group(fs.getGroup()) - .symlink(FileType.IS_SYMLINK.equals(fs.getFileType()) - ? fs.getSymlink().toByteArray() - : null) - .path(fs.getPath().toByteArray()) - .fileId(fs.hasFileId() - ? fs.getFileId() - : HdfsConstants.GRANDFATHER_INODE_ID) - .locations(fs.hasLocations() ? convert(fs.getLocations()) : null) - .children(fs.hasChildrenNum() ? fs.getChildrenNum() : -1) - .feInfo(fs.hasFileEncryptionInfo() - ? convert(fs.getFileEncryptionInfo()) - : null) - .storagePolicy(fs.hasStoragePolicy() - ? (byte) fs.getStoragePolicy() - : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) - .ecPolicy(fs.hasEcPolicy() - ? convertErasureCodingPolicy(fs.getEcPolicy()) - : null) - .build(); + return new HdfsLocatedFileStatus( + fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), + fs.getBlockReplication(), fs.getBlocksize(), + fs.getModificationTime(), fs.getAccessTime(), + convert(fs.getPermission()), + flags, + fs.getOwner(), fs.getGroup(), + fs.getFileType().equals(FileType.IS_SYMLINK) ? + fs.getSymlink().toByteArray() : null, + fs.getPath().toByteArray(), + fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID, + fs.hasLocations() ? convert(fs.getLocations()) : null, + fs.hasChildrenNum() ? fs.getChildrenNum() : -1, + fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, + fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() + : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, + fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null); } private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) { @@ -1876,10 +1864,10 @@ public class PBHelperClient { if (dl == null) return null; List<HdfsFileStatusProto> partList = dl.getPartialListingList(); - return new DirectoryListing(partList.isEmpty() - ? new HdfsFileStatus[0] - : convert(partList.toArray(new HdfsFileStatusProto[partList.size()])), - dl.getRemainingEntries()); + return new DirectoryListing(partList.isEmpty() ? + new HdfsLocatedFileStatus[0] : + convert(partList.toArray(new HdfsFileStatusProto[partList.size()])), + dl.getRemainingEntries()); } public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) { @@ -2173,9 +2161,12 @@ public class PBHelperClient { if (fs.getFileEncryptionInfo() != null) { builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo())); } - LocatedBlocks locations = fs.getLocatedBlocks(); - if (locations != null) { - builder.setLocations(convert(locations)); + if (fs instanceof HdfsLocatedFileStatus) { + final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs; + LocatedBlocks locations = lfs.getBlockLocations(); + if (locations != null) { + builder.setLocations(convert(locations)); + } } if(fs.getErasureCodingPolicy() != null) { builder.setEcPolicy(convertErasureCodingPolicy( http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml index 3f2814f..4b958b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml @@ -260,10 +260,11 @@ <Method name="visitFile" /> <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" /> </Match> - <!-- BlockLocations are user-facing, but LocatedBlocks are not. --> + <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not. + Defensible compatibility choices over time create odd corners. --> <Match> - <Class name="org.apache.hadoop.hdfs.protocol.HdfsFileStatus" /> - <Field name="hdfsloc" /> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" /> + <Field name="locations" /> <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" /> </Match> <Match> http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index aa22978..b653f4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -364,7 +364,7 @@ public class Mover { if (!isSnapshotPathInCurrent(fullPath)) { // the full path is a snapshot path but it is also included in the // current directory tree, thus ignore it. - processFile(fullPath, status, result); + processFile(fullPath, (HdfsLocatedFileStatus) status, result); } } catch (IOException e) { LOG.warn("Failed to check the status of " + parent @@ -374,7 +374,7 @@ public class Mover { } /** @return true if it is necessary to run another round of migration */ - private void processFile(String fullPath, HdfsFileStatus status, + private void processFile(String fullPath, HdfsLocatedFileStatus status, Result result) { byte policyId = status.getStoragePolicy(); if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { @@ -395,7 +395,7 @@ public class Mover { status.getReplication()); final ErasureCodingPolicy ecPolicy = status.getErasureCodingPolicy(); - final LocatedBlocks locatedBlocks = status.getLocatedBlocks(); + final LocatedBlocks locatedBlocks = status.getBlockLocations(); final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete(); List<LocatedBlock> lbs = locatedBlocks.getLocatedBlocks(); for (int i = 0; i < lbs.size(); i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index f1e2660..b6e38a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -255,13 +256,14 @@ class FSDirStatAndListingOp { listing[i] = createFileStatus(fsd, iip, child, childStoragePolicy, needLocation); listingCnt++; - LocatedBlocks blks = listing[i].getLocatedBlocks(); - if (blks != null) { - // Once we hit lsLimit locations, stop. - // This helps to prevent excessively large response payloads. - // Approximate #locations with locatedBlockCount() * repl_factor - locationBudget -= - blks.locatedBlockCount() * listing[i].getReplication(); + if (listing[i] instanceof HdfsLocatedFileStatus) { + // Once we hit lsLimit locations, stop. + // This helps to prevent excessively large response payloads. + // Approximate #locations with locatedBlockCount() * repl_factor + LocatedBlocks blks = + ((HdfsLocatedFileStatus)listing[i]).getBlockLocations(); + locationBudget -= (blks == null) ? 0 : + blks.locatedBlockCount() * listing[i].getReplication(); } } // truncate return array if necessary @@ -484,26 +486,31 @@ class FSDirStatAndListingOp { String owner, String group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy, ErasureCodingPolicy ecPolicy, LocatedBlocks locations) { - return new HdfsFileStatus.Builder() - .length(length) - .isdir(isdir) - .replication(replication) - .blocksize(blocksize) - .mtime(mtime) - .atime(atime) - .perm(permission) - .flags(flags) - .owner(owner) - .group(group) - .symlink(symlink) - .path(path) - .fileId(fileId) - .children(childrenNum) - .feInfo(feInfo) - .storagePolicy(storagePolicy) - .ecPolicy(ecPolicy) - .locations(locations) - .build(); + if (locations == null) { + return new HdfsFileStatus.Builder() + .length(length) + .isdir(isdir) + .replication(replication) + .blocksize(blocksize) + .mtime(mtime) + .atime(atime) + .perm(permission) + .flags(flags) + .owner(owner) + .group(group) + .symlink(symlink) + .path(path) + .fileId(fileId) + .children(childrenNum) + .feInfo(feInfo) + .storagePolicy(storagePolicy) + .ecPolicy(ecPolicy) + .build(); + } else { + return new HdfsLocatedFileStatus(length, isdir, replication, blocksize, + mtime, atime, permission, flags, owner, group, symlink, path, + fileId, locations, childrenNum, feInfo, storagePolicy, ecPolicy); + } } private static ContentSummary getContentSummaryInt(FSDirectory fsd, http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index bc060e1..3a8fb59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -1092,11 +1092,11 @@ public class TestBlockStoragePolicy { return types; } - private void checkLocatedBlocks(HdfsFileStatus status, int blockNum, + private void checkLocatedBlocks(HdfsLocatedFileStatus status, int blockNum, int replicaNum, StorageType... types) { List<StorageType> typeList = Lists.newArrayList(); Collections.addAll(typeList, types); - LocatedBlocks lbs = status.getLocatedBlocks(); + LocatedBlocks lbs = status.getBlockLocations(); Assert.assertEquals(blockNum, lbs.getLocatedBlocks().size()); for (LocatedBlock lb : lbs.getLocatedBlocks()) { Assert.assertEquals(replicaNum, lb.getStorageTypes().length); @@ -1127,7 +1127,7 @@ public class TestBlockStoragePolicy { HdfsFileStatus[] status = fs.getClient().listPaths(foo.toString(), HdfsFileStatus.EMPTY_NAME, true).getPartialListing(); checkDirectoryListing(status, policyId); - HdfsFileStatus fooStatus = status[0]; + HdfsLocatedFileStatus fooStatus = (HdfsLocatedFileStatus) status[0]; checkLocatedBlocks(fooStatus, 1, 3, before); // change the replication factor to 5 @@ -1140,7 +1140,7 @@ public class TestBlockStoragePolicy { status = fs.getClient().listPaths(foo.toString(), HdfsFileStatus.EMPTY_NAME, true).getPartialListing(); checkDirectoryListing(status, policyId); - fooStatus = status[0]; + fooStatus = (HdfsLocatedFileStatus) status[0]; checkLocatedBlocks(fooStatus, 1, numDataNodes, after); // change the replication factor back to 3 @@ -1157,7 +1157,7 @@ public class TestBlockStoragePolicy { status = fs.getClient().listPaths(foo.toString(), HdfsFileStatus.EMPTY_NAME, true).getPartialListing(); checkDirectoryListing(status, policyId); - fooStatus = status[0]; + fooStatus = (HdfsLocatedFileStatus) status[0]; checkLocatedBlocks(fooStatus, 1, REPLICATION, before); } finally { cluster.shutdown(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java index 3f4acba..f281a3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java @@ -274,7 +274,7 @@ public class TestDFSOutputStream { when(client.getTracer()).thenReturn(FsTracer.get(new Configuration())); client.clientRunning = true; DataStreamer stream = new DataStreamer( - new HdfsFileStatus.Builder().build(), + mock(HdfsFileStatus.class), mock(ExtendedBlock.class), client, "foo", null, null, null, null, null, null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index 0c31288..764a0db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; @@ -311,17 +312,18 @@ public class TestStorageMover { private void verifyFile(final Path parent, final HdfsFileStatus status, final Byte expectedPolicyId) throws Exception { - byte policyId = status.getStoragePolicy(); + HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus) status; + byte policyId = fileStatus.getStoragePolicy(); BlockStoragePolicy policy = policies.getPolicy(policyId); if (expectedPolicyId != null) { Assert.assertEquals((byte)expectedPolicyId, policy.getId()); } final List<StorageType> types = policy.chooseStorageTypes( status.getReplication()); - for(LocatedBlock lb : status.getLocatedBlocks().getLocatedBlocks()) { + for(LocatedBlock lb : fileStatus.getBlockLocations().getLocatedBlocks()) { final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.getStorageTypes()); - Assert.assertTrue(status.getFullName(parent.toString()) + Assert.assertTrue(fileStatus.getFullName(parent.toString()) + " with policy " + policy + " has non-empty overlap: " + diff + ", the corresponding block is " + lb.getBlock().getLocalBlock(), diff.removeOverlap(true)); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
