Merge branch 'HDFS-6584' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98f7bfe9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98f7bfe9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98f7bfe9 Branch: refs/heads/HDFS-6581 Commit: 98f7bfe99395bb27d97a4f6076fc10ede50383f0 Parents: fad4cd8 76ccb80 Author: Tsz-Wo Nicholas Sze <[email protected]> Authored: Fri Sep 19 07:11:19 2014 +0800 Committer: Tsz-Wo Nicholas Sze <[email protected]> Committed: Fri Sep 19 07:11:19 2014 +0800 ---------------------------------------------------------------------- .../hadoop-common/src/main/conf/hadoop-env.sh | 9 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 76 ++ .../hadoop-hdfs/src/main/bin/hdfs | 7 + .../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +- .../src/main/conf/blockStoragePolicy-site.xml | 21 + .../hadoop-hdfs/src/main/conf/hdfs-site.xml | 3 +- .../apache/hadoop/hdfs/BlockStoragePolicy.java | 419 +++++++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 19 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 15 + .../hadoop/hdfs/DistributedFileSystem.java | 34 +- .../org/apache/hadoop/hdfs/StorageType.java | 6 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 14 + .../hadoop/hdfs/protocol/HdfsConstants.java | 5 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 10 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 10 +- .../protocol/SnapshottableDirectoryStatus.java | 3 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 16 + .../ClientNamenodeProtocolTranslatorPB.java | 16 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 37 +- .../hadoop/hdfs/server/balancer/Balancer.java | 19 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 241 ++-- .../hadoop/hdfs/server/balancer/Matcher.java | 15 + .../hdfs/server/balancer/MovedBlocks.java | 2 +- .../hdfs/server/balancer/NameNodeConnector.java | 79 +- .../server/blockmanagement/BlockCollection.java | 5 + .../server/blockmanagement/BlockManager.java | 89 +- .../blockmanagement/BlockPlacementPolicy.java | 22 +- .../BlockPlacementPolicyDefault.java | 377 +++--- .../BlockPlacementPolicyWithNodeGroup.java | 73 +- .../blockmanagement/DatanodeDescriptor.java | 52 +- .../blockmanagement/DatanodeStorageInfo.java | 24 +- .../hadoop/hdfs/server/datanode/DataNode.java | 4 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 644 +++++++++++ .../hdfs/server/namenode/FSDirectory.java | 118 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 12 +- .../hdfs/server/namenode/FSEditLogLoader.java | 12 +- .../hdfs/server/namenode/FSEditLogOp.java | 67 ++ .../hdfs/server/namenode/FSEditLogOpCodes.java | 1 + .../hdfs/server/namenode/FSImageFormat.java | 4 +- .../server/namenode/FSImageFormatPBINode.java | 6 +- .../server/namenode/FSImageSerialization.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 89 +- .../hadoop/hdfs/server/namenode/INode.java | 17 +- .../hdfs/server/namenode/INodeDirectory.java | 27 + .../namenode/INodeDirectoryAttributes.java | 4 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 49 +- .../server/namenode/INodeFileAttributes.java | 14 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 11 + .../hdfs/server/namenode/INodeReference.java | 10 + .../hdfs/server/namenode/INodeSymlink.java | 12 + .../server/namenode/NameNodeLayoutVersion.java | 3 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 +- .../snapshot/FSImageFormatPBSnapshot.java | 3 +- .../web/resources/NamenodeWebHdfsMethods.java | 8 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 75 +- .../apache/hadoop/hdfs/util/EnumCounters.java | 9 + .../org/apache/hadoop/hdfs/web/JsonUtil.java | 10 +- .../src/main/proto/ClientNamenodeProtocol.proto | 10 + .../hadoop-hdfs/src/main/proto/fsimage.proto | 1 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 3 + .../resources/blockStoragePolicy-default.xml | 118 ++ .../src/main/resources/hdfs-default.xml | 3 +- .../src/site/apt/ArchivalStorage.apt.vm | 302 +++++ .../src/site/apt/HDFSCommands.apt.vm | 43 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 119 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 12 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 1075 ++++++++++++++++++ .../hadoop/hdfs/TestDFSClientRetries.java | 4 +- .../hdfs/TestDFSInotifyEventInputStream.java | 2 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 9 +- .../hadoop/hdfs/TestSnapshotCommands.java | 152 +-- .../hadoop/hdfs/TestStoragePolicyCommands.java | 81 ++ .../hdfs/server/balancer/TestBalancer.java | 7 + .../balancer/TestBalancerWithHANameNodes.java | 2 +- .../TestBalancerWithMultipleNameNodes.java | 2 +- .../balancer/TestBalancerWithNodeGroup.java | 2 +- .../blockmanagement/TestBlockManager.java | 20 +- .../blockmanagement/TestReplicationPolicy.java | 27 +- .../TestReplicationPolicyConsiderLoad.java | 28 +- .../TestReplicationPolicyWithNodeGroup.java | 17 +- .../hadoop/hdfs/server/mover/TestMover.java | 222 ++++ .../hdfs/server/mover/TestStorageMover.java | 766 +++++++++++++ .../hdfs/server/namenode/CreateEditsLog.java | 4 +- .../hdfs/server/namenode/TestAddBlockRetry.java | 4 +- .../hdfs/server/namenode/TestDeleteRace.java | 6 +- .../hdfs/server/namenode/TestEditLog.java | 2 +- .../namenode/TestFSPermissionChecker.java | 35 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 3 +- .../hdfs/server/namenode/TestINodeFile.java | 35 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 4 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +- .../hadoop-hdfs/src/test/resources/editsStored | Bin 5252 -> 4999 bytes .../src/test/resources/editsStored.xml | 422 ++++--- hadoop-project/src/site/site.xml | 1 + 94 files changed, 5666 insertions(+), 824 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/98f7bfe9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/98f7bfe9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ----------------------------------------------------------------------
