Merge r1609845 through r1619277 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-6584@1619293 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c92d869d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c92d869d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c92d869d Branch: refs/heads/trunk Commit: c92d869d02af4d7870649317474134ba94b1dd0c Parents: 4e324bc 6824abc Author: Tsz-wo Sze <szets...@apache.org> Authored: Thu Aug 21 05:22:10 2014 +0000 Committer: Tsz-wo Sze <szets...@apache.org> Committed: Thu Aug 21 05:22:10 2014 +0000 ---------------------------------------------------------------------- BUILDING.txt | 21 + .../main/resources/assemblies/hadoop-dist.xml | 7 + hadoop-common-project/hadoop-common/CHANGES.txt | 57 +- hadoop-common-project/hadoop-common/pom.xml | 19 +- .../hadoop-common/src/CMakeLists.txt | 34 + .../hadoop-common/src/JNIFlags.cmake | 6 + .../hadoop-common/src/config.h.cmake | 1 + .../hadoop-common/src/main/bin/hadoop | 230 ++-- .../hadoop-common/src/main/bin/hadoop-config.sh | 396 +++---- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 214 +--- .../src/main/bin/hadoop-daemons.sh | 37 +- .../src/main/bin/hadoop-functions.sh | 1036 ++++++++++++++++++ .../src/main/bin/hadoop-layout.sh.example | 93 ++ .../hadoop-common/src/main/bin/rcc | 51 +- .../hadoop-common/src/main/bin/slaves.sh | 51 +- .../hadoop-common/src/main/bin/start-all.sh | 38 +- .../hadoop-common/src/main/bin/stop-all.sh | 36 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 421 ++++++- .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 ++ .../org/apache/hadoop/crypto/CipherSuite.java | 115 ++ .../org/apache/hadoop/crypto/CryptoCodec.java | 174 +++ .../apache/hadoop/crypto/CryptoInputStream.java | 680 ++++++++++++ .../hadoop/crypto/CryptoOutputStream.java | 280 +++++ .../apache/hadoop/crypto/CryptoStreamUtils.java | 70 ++ .../org/apache/hadoop/crypto/Decryptor.java | 72 ++ .../org/apache/hadoop/crypto/Encryptor.java | 71 ++ .../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 +++ .../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 +++ .../org/apache/hadoop/crypto/OpensslCipher.java | 287 +++++ .../crypto/random/OpensslSecureRandom.java | 119 ++ .../hadoop/crypto/random/OsSecureRandom.java | 115 ++ .../hadoop/fs/CommonConfigurationKeys.java | 1 - .../fs/CommonConfigurationKeysPublic.java | 30 + .../apache/hadoop/fs/FSDataOutputStream.java | 2 +- .../apache/hadoop/fs/FileEncryptionInfo.java | 102 ++ .../fs/crypto/CryptoFSDataInputStream.java | 37 + .../fs/crypto/CryptoFSDataOutputStream.java | 47 + .../hadoop/fs/shell/CommandWithDestination.java | 75 +- .../apache/hadoop/fs/shell/CopyCommands.java | 6 +- .../apache/hadoop/util/NativeCodeLoader.java | 5 + .../hadoop/util/NativeLibraryChecker.java | 21 +- .../org/apache/hadoop/crypto/OpensslCipher.c | 382 +++++++ .../hadoop/crypto/org_apache_hadoop_crypto.h | 61 ++ .../hadoop/crypto/random/OpensslSecureRandom.c | 335 ++++++ .../random/org_apache_hadoop_crypto_random.h | 40 + .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../src/main/resources/core-default.xml | 69 ++ .../src/site/apt/FileSystemShell.apt.vm | 11 +- .../hadoop/crypto/CryptoStreamsTestBase.java | 721 ++++++++++++ .../apache/hadoop/crypto/TestCryptoCodec.java | 186 ++++ .../apache/hadoop/crypto/TestCryptoStreams.java | 376 +++++++ .../crypto/TestCryptoStreamsForLocalFS.java | 120 ++ .../hadoop/crypto/TestCryptoStreamsNormal.java | 123 +++ ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 31 + .../apache/hadoop/crypto/TestOpensslCipher.java | 110 ++ .../crypto/random/TestOpensslSecureRandom.java | 114 ++ .../crypto/random/TestOsSecureRandom.java | 139 +++ .../hadoop/util/TestNativeCodeLoader.java | 4 + .../src/test/resources/testConf.xml | 18 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 108 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../src/main/bin/distribute-exclude.sh | 6 +- .../hadoop-hdfs/src/main/bin/hdfs | 417 ++++--- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 68 +- .../src/main/bin/refresh-namenodes.sh | 40 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 30 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 142 ++- .../src/main/bin/start-secure-dns.sh | 34 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 30 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 105 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 34 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 20 +- .../main/java/org/apache/hadoop/fs/XAttr.java | 13 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 158 ++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 40 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 38 + .../hadoop/hdfs/DistributedFileSystem.java | 52 +- .../hdfs/UnknownCipherSuiteException.java | 38 + .../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 50 + .../hadoop/hdfs/client/HdfsDataInputStream.java | 38 +- .../hdfs/client/HdfsDataOutputStream.java | 36 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 30 +- .../hadoop/hdfs/protocol/EncryptionZone.java | 79 ++ .../hdfs/protocol/EncryptionZoneIterator.java | 51 + .../hdfs/protocol/EncryptionZoneWithId.java | 64 ++ .../protocol/EncryptionZoneWithIdIterator.java | 53 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 12 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 6 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 23 +- .../protocol/SnapshottableDirectoryStatus.java | 2 +- .../datatransfer/sasl/DataTransferSaslUtil.java | 4 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 56 +- .../ClientNamenodeProtocolTranslatorPB.java | 81 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 102 +- .../server/blockmanagement/BlockManager.java | 9 +- .../hdfs/server/common/HdfsServerConstants.java | 5 + .../hdfs/server/datanode/DataXceiver.java | 4 +- .../hdfs/server/datanode/DataXceiverServer.java | 7 +- .../namenode/EncryptionFaultInjector.java | 22 + .../server/namenode/EncryptionZoneManager.java | 296 +++++ .../hdfs/server/namenode/FSDirectory.java | 313 +++++- .../hdfs/server/namenode/FSEditLogLoader.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 655 ++++++++--- .../hdfs/server/namenode/NameNodeRpcServer.java | 25 +- .../namenode/RetryStartFileException.java | 21 + .../server/namenode/XAttrPermissionFilter.java | 30 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 301 +++++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 4 +- .../src/main/proto/ClientNamenodeProtocol.proto | 8 + .../hadoop-hdfs/src/main/proto/encryption.proto | 65 ++ .../hadoop-hdfs/src/main/proto/hdfs.proto | 27 +- .../hadoop-hdfs/src/main/proto/xattr.proto | 1 + .../src/main/resources/hdfs-default.xml | 16 +- .../src/site/apt/ExtendedAttributes.apt.vm | 4 +- .../src/site/apt/HdfsNfsGateway.apt.vm | 4 +- .../src/site/apt/TransparentEncryption.apt.vm | 206 ++++ .../apache/hadoop/cli/TestCryptoAdminCLI.java | 169 +++ .../hadoop/cli/util/CLICommandCryptoAdmin.java | 21 + .../hadoop/cli/util/CryptoAdminCmdExecutor.java | 37 + .../java/org/apache/hadoop/fs/TestXAttr.java | 11 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 68 ++ .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 82 +- .../hdfs/MiniDFSClusterWithNodeGroup.java | 12 +- .../hadoop/hdfs/TestDFSClientRetries.java | 11 +- .../org/apache/hadoop/hdfs/TestDFSRename.java | 44 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 177 ++- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 1 - .../apache/hadoop/hdfs/TestEncryptionZones.java | 756 +++++++++++++ .../apache/hadoop/hdfs/TestFileCreation.java | 2 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 11 +- .../hadoop/hdfs/TestReservedRawPaths.java | 350 ++++++ .../hdfs/TestWriteBlockGetsBlockLengthHint.java | 106 ++ .../hdfs/crypto/TestHdfsCryptoStreams.java | 91 ++ .../hdfs/server/datanode/TestStorageReport.java | 2 +- .../hdfs/server/namenode/FSXAttrBaseTest.java | 179 ++- .../server/namenode/NNThroughputBenchmark.java | 5 +- .../hdfs/server/namenode/TestAddBlockRetry.java | 4 +- .../hdfs/server/namenode/TestFSDirectory.java | 15 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 2 +- .../server/namenode/TestNamenodeRetryCache.java | 9 +- .../namenode/ha/TestRetryCacheWithHA.java | 2 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +- .../src/test/resources/testCryptoConf.xml | 284 +++++ .../src/test/resources/testXAttrConf.xml | 58 +- hadoop-mapreduce-project/CHANGES.txt | 15 +- hadoop-mapreduce-project/bin/mapred | 222 ++-- hadoop-mapreduce-project/bin/mapred-config.sh | 72 +- .../bin/mr-jobhistory-daemon.sh | 143 +-- hadoop-mapreduce-project/conf/mapred-env.sh | 60 +- .../org/apache/hadoop/mapred/BackupStore.java | 6 +- .../java/org/apache/hadoop/mapred/IFile.java | 21 +- .../java/org/apache/hadoop/mapred/MapTask.java | 29 +- .../java/org/apache/hadoop/mapred/Merger.java | 15 +- .../apache/hadoop/mapreduce/CryptoUtils.java | 199 ++++ .../apache/hadoop/mapreduce/JobSubmitter.java | 9 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 14 + .../hadoop/mapreduce/task/reduce/Fetcher.java | 12 +- .../mapreduce/task/reduce/LocalFetcher.java | 4 + .../mapreduce/task/reduce/MergeManagerImpl.java | 32 +- .../mapreduce/task/reduce/OnDiskMapOutput.java | 3 +- .../src/site/apt/MapredCommands.apt.vm | 6 + .../src/site/markdown/DistCp.md.vm | 20 + .../mapreduce/task/reduce/TestMerger.java | 146 ++- .../org/apache/hadoop/mapred/TestIFile.java | 11 +- .../TestMRIntermediateDataEncryption.java | 254 +++++ .../apache/hadoop/mapred/TestReduceTask.java | 2 +- .../mapred/pipes/TestPipeApplication.java | 12 +- hadoop-project-dist/pom.xml | 12 + hadoop-project/pom.xml | 2 + hadoop-project/src/site/site.xml | 1 + .../apache/hadoop/tools/DistCpConstants.java | 6 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 6 +- .../org/apache/hadoop/tools/DistCpOptions.java | 21 +- .../apache/hadoop/tools/SimpleCopyListing.java | 48 +- .../hadoop/tools/mapred/CopyCommitter.java | 9 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 10 +- .../apache/hadoop/tools/util/DistCpUtils.java | 48 +- .../hadoop/tools/TestDistCpWithRawXAttrs.java | 170 +++ .../hadoop/tools/TestDistCpWithXAttrs.java | 71 +- .../apache/hadoop/tools/TestOptionsParser.java | 3 +- .../hadoop/tools/util/DistCpTestUtils.java | 89 ++ .../hadoop/tools/util/TestDistCpUtils.java | 6 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 9 +- hadoop-yarn-project/CHANGES.txt | 17 +- hadoop-yarn-project/hadoop-yarn/bin/slaves.sh | 70 -- .../hadoop-yarn/bin/start-yarn.sh | 34 +- .../hadoop-yarn/bin/stop-yarn.sh | 40 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 380 +++---- .../hadoop-yarn/bin/yarn-config.sh | 116 +- .../hadoop-yarn/bin/yarn-daemon.sh | 157 +-- .../hadoop-yarn/bin/yarn-daemons.sh | 38 +- .../hadoop-yarn/conf/yarn-env.sh | 163 ++- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +- .../client/api/impl/TimelineAuthenticator.java | 15 +- .../client/api/impl/TimelineClientImpl.java | 98 +- .../src/main/resources/yarn-default.xml | 5 +- .../webapp/TestTimelineWebServicesWithSSL.java | 134 +++ .../resourcemanager/EmbeddedElectorService.java | 10 +- .../server/resourcemanager/ResourceManager.java | 3 + .../scheduler/AbstractYarnScheduler.java | 91 ++ .../scheduler/SchedulerApplicationAttempt.java | 18 +- .../scheduler/capacity/CapacityScheduler.java | 23 +- .../scheduler/fair/FairScheduler.java | 21 +- .../scheduler/fifo/FifoScheduler.java | 30 +- .../yarn/server/resourcemanager/MockAM.java | 8 +- .../TestApplicationMasterService.java | 1 - .../yarn/server/resourcemanager/TestRMHA.java | 23 + .../server/resourcemanager/TestRMRestart.java | 4 +- .../TestWorkPreservingRMRestart.java | 79 +- 213 files changed, 15760 insertions(+), 2565 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 9a9e1b6,3d05639..94d9a92 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@@ -45,9 -46,10 +46,11 @@@ public class HdfsFileStatus private final String group; private final long fileId; + private final FileEncryptionInfo feInfo; + // Used by dir, not including dot and dotdot. Always zero for a regular file. private final int childrenNum; + private final byte storagePolicy; public static final byte[] EMPTY_NAME = new byte[0]; @@@ -68,7 -71,7 +72,8 @@@ public HdfsFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, - byte[] path, long fileId, int childrenNum, byte storagePolicy) { - byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo) { ++ byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, ++ byte storagePolicy) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@@ -86,7 -89,7 +91,8 @@@ this.path = path; this.fileId = fileId; this.childrenNum = childrenNum; + this.feInfo = feInfo; + this.storagePolicy = storagePolicy; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 7f0e670,a78b8bc..7e602bf --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@@ -56,13 -58,13 +58,13 @@@ public class HdfsLocatedFileStatus exte int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, LocatedBlocks locations, - int childrenNum, byte storagePolicy) { - int childrenNum, FileEncryptionInfo feInfo) { ++ int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) { super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, owner, group, symlink, path, fileId, - childrenNum, feInfo); + access_time, permission, owner, group, symlink, path, fileId, - childrenNum, storagePolicy); ++ childrenNum, feInfo, storagePolicy); this.locations = locations; } - + public LocatedBlocks getBlockLocations() { return locations; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java index 7869826,d395283..13acc7a --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java @@@ -62,7 -61,7 +62,7 @@@ public class SnapshottableDirectoryStat int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, access_time, permission, owner, group, null, localName, inodeId, - childrenNum, BlockStoragePolicy.ID_UNSPECIFIED); - childrenNum, null); ++ childrenNum, null, BlockStoragePolicy.ID_UNSPECIFIED); this.snapshotNumber = snapshotNumber; this.snapshotQuota = snapshotQuota; this.parentFullPath = parentFullPath; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index adfcf36,210828d..63af6b0 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@@ -148,7 -149,10 +151,11 @@@ import org.apache.hadoop.hdfs.protocol. import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto; + import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos; + import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto; + import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto; + import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 9957008,4dcac39..136ebc8 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@@ -1317,8 -1326,8 +1327,9 @@@ public class PBHelper fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID, fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null, fs.hasChildrenNum() ? fs.getChildrenNum() : -1, - fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : - null); ++ fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, + fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() + : BlockStoragePolicy.ID_UNSPECIFIED); } public static SnapshottableDirectoryStatus convert( @@@ -1369,9 -1377,11 +1380,12 @@@ if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); } + if (fs.getFileEncryptionInfo() != null) { + builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo())); + } if (fs instanceof HdfsLocatedFileStatus) { - LocatedBlocks locations = ((HdfsLocatedFileStatus)fs).getBlockLocations(); + final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs; + LocatedBlocks locations = lfs.getBlockLocations(); if (locations != null) { builder.setLocations(PBHelper.convert(locations)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 860855f,54e3181..4d0bb86 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@@ -1330,9 -1314,9 +1357,10 @@@ public class FSDirectory implements Clo * @return a partial listing starting after startAfter */ DirectoryListing getListing(String src, byte[] startAfter, - boolean needLocation) throws UnresolvedLinkException, IOException { + boolean needLocation, boolean isSuperUser) + throws UnresolvedLinkException, IOException { String srcs = normalizePath(src); + final boolean isRawPath = isReservedRawName(src); readLock(); try { @@@ -1351,7 -1332,7 +1379,8 @@@ if (!targetNode.isDirectory()) { return new DirectoryListing( new HdfsFileStatus[]{createFileStatus(HdfsFileStatus.EMPTY_NAME, - targetNode, needLocation, parentStoragePolicy, snapshot)}, 0); - targetNode, needLocation, snapshot, isRawPath)}, 0); ++ targetNode, needLocation, parentStoragePolicy, snapshot, ++ isRawPath)}, 0); } final INodeDirectory dirInode = targetNode.asDirectory(); @@@ -1364,10 -1345,8 +1393,11 @@@ HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; for (int i=0; i<numOfListing && locationBudget>0; i++) { INode cur = contents.get(startChild+i); - listing[i] = createFileStatus(cur.getLocalNameBytes(), cur, - needLocation, snapshot, isRawPath); + byte curPolicy = cur.getStoragePolicyID(snapshot); - listing[i] = createFileStatus(cur.getLocalNameBytes(), cur, - needLocation, curPolicy != BlockStoragePolicy.ID_UNSPECIFIED ? - curPolicy : parentStoragePolicy, snapshot); ++ listing[i] = createFileStatus(cur.getLocalNameBytes(), cur, needLocation, ++ curPolicy != BlockStoragePolicy.ID_UNSPECIFIED ? ++ curPolicy : parentStoragePolicy, ++ snapshot, isRawPath); listingCnt++; if (needLocation) { // Once we hit lsLimit locations, stop. @@@ -1418,7 -1397,7 +1448,7 @@@ for (int i = 0; i < numOfListing; i++) { Root sRoot = snapshots.get(i + skipSize).getRoot(); listing[i] = createFileStatus(sRoot.getLocalNameBytes(), sRoot, - BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID); - Snapshot.CURRENT_STATE_ID, false); ++ BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, false); } return new DirectoryListing( listing, snapshots.size() - skipSize - numOfListing); @@@ -1440,8 -1420,9 +1471,9 @@@ } final INodesInPath inodesInPath = getLastINodeInPath(srcs, resolveLink); final INode i = inodesInPath.getINode(0); - - return i == null? null: createFileStatus(HdfsFileStatus.EMPTY_NAME, i, - inodesInPath.getPathSnapshotId(), isRawPath); + return i == null ? null : createFileStatus(HdfsFileStatus.EMPTY_NAME, i, - BlockStoragePolicy.ID_UNSPECIFIED, inodesInPath.getPathSnapshotId()); ++ BlockStoragePolicy.ID_UNSPECIFIED, inodesInPath.getPathSnapshotId(), ++ isRawPath); } finally { readUnlock(); } @@@ -1458,7 -1439,7 +1490,7 @@@ throws UnresolvedLinkException { if (getINode4DotSnapshot(src) != null) { return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null, - HdfsFileStatus.EMPTY_NAME, -1L, 0, BlockStoragePolicy.ID_UNSPECIFIED); - HdfsFileStatus.EMPTY_NAME, -1L, 0, null); ++ HdfsFileStatus.EMPTY_NAME, -1L, 0, null, BlockStoragePolicy.ID_UNSPECIFIED); } return null; } @@@ -2270,19 -2267,19 +2318,20 @@@ * @throws IOException if any error occurs */ private HdfsFileStatus createFileStatus(byte[] path, INode node, - boolean needLocation, byte storagePolicy, int snapshot) throws IOException { - boolean needLocation, int snapshot, boolean isRawPath) ++ boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath) + throws IOException { if (needLocation) { - return createLocatedFileStatus(path, node, storagePolicy, snapshot); - return createLocatedFileStatus(path, node, snapshot, isRawPath); ++ return createLocatedFileStatus(path, node, storagePolicy, snapshot, isRawPath); } else { - return createFileStatus(path, node, storagePolicy, snapshot); - return createFileStatus(path, node, snapshot, isRawPath); ++ return createFileStatus(path, node, storagePolicy, snapshot, isRawPath); } } + /** * Create FileStatus by file INode */ - HdfsFileStatus createFileStatus(byte[] path, INode node, - int snapshot, boolean isRawPath) throws IOException { + HdfsFileStatus createFileStatus(byte[] path, INode node, byte storagePolicy, - int snapshot) { ++ int snapshot, boolean isRawPath) throws IOException { long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; @@@ -2294,6 -2291,9 +2343,8 @@@ } int childrenNum = node.isDirectory() ? node.asDirectory().getChildrenNum(snapshot) : 0; - + FileEncryptionInfo feInfo = isRawPath ? null : + getFileEncryptionInfo(node, snapshot); return new HdfsFileStatus( size, @@@ -2308,24 -2308,15 +2359,26 @@@ node.isSymlink() ? node.asSymlink().getSymlink() : null, path, node.getId(), - childrenNum, storagePolicy); + childrenNum, - feInfo); ++ feInfo, ++ storagePolicy); } + private byte getStoragePolicy(INode[] inodes, int snapshotId) { + for (int i = inodes.length - 1; i >= 0; i--) { + byte policy = inodes[i].getStoragePolicyID(snapshotId); + if (policy != BlockStoragePolicy.ID_UNSPECIFIED) { + return policy; + } + } + return BlockStoragePolicy.ID_UNSPECIFIED; + } + /** * Create FileStatus with location info by file INode */ -- private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path, - INode node, byte storagePolicy, int snapshot) throws IOException { - INode node, int snapshot, boolean isRawPath) throws IOException { ++ private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path, INode node, ++ byte storagePolicy, int snapshot, boolean isRawPath) throws IOException { assert hasReadLock(); long size = 0; // length is zero for directories short replication = 0; @@@ -2358,7 -2352,7 +2414,7 @@@ getPermissionForFileStatus(node, snapshot), node.getUserName(snapshot), node.getGroupName(snapshot), node.isSymlink() ? node.asSymlink().getSymlink() : null, path, - node.getId(), loc, childrenNum, storagePolicy); - node.getId(), loc, childrenNum, feInfo); ++ node.getId(), loc, childrenNum, feInfo, storagePolicy); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = namesystem.getCacheManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index fbb330c,d522e51..6595dd5 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@@ -366,8 -364,8 +366,9 @@@ public class FSEditLogLoader // add the op into retry cache if necessary if (toAddRetryCache) { HdfsFileStatus stat = fsNamesys.dir.createFileStatus( - HdfsFileStatus.EMPTY_NAME, newFile, Snapshot.CURRENT_STATE_ID, + HdfsFileStatus.EMPTY_NAME, newFile, - BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID); ++ BlockStoragePolicy.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, + false); fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, addCloseOp.rpcCallId, stat); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8df0776,81d5a22..4e1bbe1 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@@ -159,6 -169,8 +170,7 @@@ import org.apache.hadoop.hdfs.DFSConfig import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.StorageType; + import org.apache.hadoop.hdfs.UnknownCipherSuiteException; import org.apache.hadoop.hdfs.protocol.AclException; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.Block; @@@ -4560,10 -4740,9 +4791,10 @@@ public class FSNamesystem implements Na } else { checkTraverse(pc, src); } + isSuperUser = pc.isSuperUser(); } - logAuditEvent(true, "listStatus", src); + logAuditEvent(true, "listStatus", srcArg); - dl = dir.getListing(src, startAfter, needLocation); + dl = dir.getListing(src, startAfter, needLocation, isSuperUser); } finally { readUnlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 1ba7805,321630c..735191b --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@@ -252,12 -250,10 +252,12 @@@ public class JsonUtil Long childrenNumLong = (Long) m.get("childrenNum"); final int childrenNum = (childrenNumLong == null) ? -1 : childrenNumLong.intValue(); + final byte storagePolicy = m.containsKey("storagePolicy") ? + (byte) (long) (Long) m.get("storagePolicy") : + BlockStoragePolicy.ID_UNSPECIFIED; return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication, - blockSize, mTime, aTime, permission, owner, group, - symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum, - null); + blockSize, mTime, aTime, permission, owner, group, symlink, - DFSUtil.string2Bytes(localName), fileId, childrenNum, storagePolicy); ++ DFSUtil.string2Bytes(localName), fileId, childrenNum, null, storagePolicy); } /** Convert an ExtendedBlock to a Json map. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 4b64591,a410224..f12055c --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@@ -244,8 -260,10 +261,12 @@@ message HdfsFileStatusProto // Optional field for fileId optional uint64 fileId = 13 [default = 0]; // default as an invalid id optional int32 childrenNum = 14 [default = -1]; - optional uint32 storagePolicy = 15 [default = 0]; // block storage policy id - } + + // Optional field for file encryption + optional FileEncryptionInfoProto fileEncryptionInfo = 15; ++ ++ optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id + } /** * Checksum algorithms/types used in HDFS http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 2942ecb,74daccc..5659e92 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@@ -253,12 -255,12 +255,12 @@@ public class TestDFSClientRetries Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)).when(mockNN).getFileInfo(anyString()); - 1010, 0, null)).when(mockNN).getFileInfo(anyString()); ++ 1010, 0, null, (byte) 0)).when(mockNN).getFileInfo(anyString()); Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)) - 1010, 0, null)) ++ 1010, 0, null, (byte) 0)) .when(mockNN) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java index 489f9ff,28c253f..6119b6e --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@@ -17,11 -17,12 +17,14 @@@ */ package org.apache.hadoop.hdfs; +import static org.mockito.Matchers.anyBoolean; + import static org.mockito.Matchers.anyList; + import static org.mockito.Matchers.anyString; + import static org.mockito.Matchers.anyShort; import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anyShort; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; @@@ -33,8 -35,10 +37,9 @@@ import java.util.List import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; @@@ -338,12 -342,12 +343,12 @@@ public class TestLease Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)).when(mcp).getFileInfo(anyString()); - 1010, 0, null)).when(mcp).getFileInfo(anyString()); ++ 1010, 0, null, (byte) 0)).when(mcp).getFileInfo(anyString()); Mockito .doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)) - 1010, 0, null)) ++ 1010, 0, null, (byte) 0)) .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 4bf1512,4cddd60..8d298ae --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@@ -1019,7 -1018,7 +1019,7 @@@ public class TestFsck HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, - fileId, numChildren, storagePolicy); - fileId, numChildren, null); ++ fileId, numChildren, null, storagePolicy); Result res = new Result(conf); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c92d869d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java index a268128,b8150f7..3eba7db --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java @@@ -64,7 -64,7 +64,7 @@@ public class TestJsonUtil final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26, now, now + 10, new FsPermission((short) 0644), "user", "group", DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"), - INodeId.GRANDFATHER_INODE_ID, 0, (byte) 0); - INodeId.GRANDFATHER_INODE_ID, 0, null); ++ INodeId.GRANDFATHER_INODE_ID, 0, null, (byte) 0); final FileStatus fstatus = toFileStatus(status, parent); System.out.println("status = " + status); System.out.println("fstatus = " + fstatus);
