Merge remote-tracking branch 'apache/trunk' into HDFS-7285
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Change-Id: I8511c4d64b0959e79129febc179845a3892fedcc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1080c373
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1080c373
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1080c373
Branch: refs/heads/HDFS-7240
Commit: 1080c3730068177ddd10dc313890ac1f5dc58f1a
Parents: a9e6681 dfd807a
Author: Zhe Zhang <zhezh...@cloudera.com>
Authored: Mon Sep 21 13:51:36 2015 -0700
Committer: Zhe Zhang <zhezh...@cloudera.com>
Committed: Tue Sep 22 13:27:09 2015 -0700
----------------------------------------------------------------------
.../main/resources/assemblies/hadoop-tools.xml | 7 +
.../server/AuthenticationToken.java | 3 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 205 +-
.../src/main/conf/log4j.properties | 17 +-
.../org/apache/hadoop/conf/Configuration.java | 11 +-
.../fs/CommonConfigurationKeysPublic.java | 6 +
.../java/org/apache/hadoop/fs/FileUtil.java | 14 +-
.../main/java/org/apache/hadoop/fs/Trash.java | 3 +-
.../hadoop/fs/shell/CommandWithDestination.java | 33 +-
.../apache/hadoop/fs/shell/CopyCommands.java | 19 +-
.../java/org/apache/hadoop/fs/shell/Delete.java | 68 +-
.../apache/hadoop/ha/ActiveStandbyElector.java | 4 +
.../apache/hadoop/io/retry/RetryPolicies.java | 13 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 9 +-
.../hadoop/metrics2/impl/MetricsConfig.java | 3 +
.../metrics2/impl/MetricsSourceAdapter.java | 12 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 20 +-
.../apache/hadoop/metrics2/package-info.java | 4 +-
.../apache/hadoop/metrics2/sink/StatsDSink.java | 218 ++
.../apache/hadoop/security/SaslPlainServer.java | 2 +-
.../security/authorize/AccessControlList.java | 2 +-
.../java/org/apache/hadoop/util/LineReader.java | 17 +-
.../org/apache/hadoop/util/VersionInfo.java | 2 +-
.../src/main/resources/core-default.xml | 11 +
.../src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/site/markdown/Metrics.md | 3 +
.../java/org/apache/hadoop/cli/TestCLI.java | 2 +-
.../org/apache/hadoop/cli/util/CLICommand.java | 5 +-
.../org/apache/hadoop/cli/util/CLITestCmd.java | 6 +-
.../org/apache/hadoop/fs/TestFsShellCopy.java | 46 +
.../apache/hadoop/io/retry/TestRetryProxy.java | 13 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 38 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 9 +-
.../metrics2/impl/TestMetricsSystemImpl.java | 13 +
.../hadoop/metrics2/impl/TestStatsDMetrics.java | 122 +
.../authorize/TestAccessControlList.java | 9 +
.../delegation/web/TestWebDelegationToken.java | 7 +-
.../src/test/resources/testConf.xml | 22 +-
.../dev-support/findbugsExcludeFile.xml | 1 +
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 105 +
.../hdfs/client/HdfsClientConfigKeys.java | 14 +
.../apache/hadoop/hdfs/net/BasicInetPeer.java | 133 +
.../apache/hadoop/hdfs/net/EncryptedPeer.java | 142 ++
.../org/apache/hadoop/hdfs/net/NioInetPeer.java | 136 +
.../java/org/apache/hadoop/hdfs/net/Peer.java | 8 +-
.../hdfs/protocol/ClientDatanodeProtocol.java | 7 +
.../hadoop/hdfs/protocol/DatanodeInfo.java | 47 +-
.../hadoop/hdfs/protocol/HdfsConstants.java | 8 +
.../hdfs/protocol/HdfsLocatedFileStatus.java | 84 +
.../protocol/datatransfer/IOStreamPair.java | 37 +
.../hdfs/protocol/datatransfer/PipelineAck.java | 243 ++
.../datatransfer/TrustedChannelResolver.java | 81 +
.../sasl/DataEncryptionKeyFactory.java | 38 +
.../datatransfer/sasl/DataTransferSaslUtil.java | 519 ++++
.../sasl/SaslDataTransferClient.java | 498 ++++
.../datatransfer/sasl/SaslParticipant.java | 210 ++
.../SaslResponseWithNegotiatedCipherOption.java | 33 +
.../ClientDatanodeProtocolTranslatorPB.java | 17 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 2186 ++++++++++++++++-
.../apache/hadoop/hdfs/util/LongBitFormat.java | 71 +
.../hadoop/hdfs/web/ByteRangeInputStream.java | 11 +
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 3 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 4 +-
.../src/main/proto/ClientDatanodeProtocol.proto | 16 +
.../src/main/proto/hdfs.proto | 1 +
.../services/org.apache.hadoop.fs.FileSystem | 17 +
.../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 4 +-
.../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 4 +-
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 7 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 409 ++-
.../hdfs/server/namenode/FSEditLogTestUtil.java | 3 +-
.../main/java/org/apache/hadoop/fs/Hdfs.java | 7 +-
.../java/org/apache/hadoop/fs/SWebHdfs.java | 15 +-
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 14 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 50 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 10 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 8 +-
.../hadoop/hdfs/DistributedFileSystem.java | 4 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 15 +-
.../apache/hadoop/hdfs/net/BasicInetPeer.java | 133 -
.../hadoop/hdfs/net/DomainPeerServer.java | 5 +
.../apache/hadoop/hdfs/net/EncryptedPeer.java | 142 --
.../org/apache/hadoop/hdfs/net/NioInetPeer.java | 136 -
.../org/apache/hadoop/hdfs/net/PeerServer.java | 9 +-
.../apache/hadoop/hdfs/net/TcpPeerServer.java | 70 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 84 -
.../protocol/datatransfer/IOStreamPair.java | 37 -
.../hdfs/protocol/datatransfer/PipelineAck.java | 274 ---
.../hdfs/protocol/datatransfer/Receiver.java | 33 +-
.../datatransfer/TrustedChannelResolver.java | 81 -
.../sasl/DataEncryptionKeyFactory.java | 38 -
.../datatransfer/sasl/DataTransferSaslUtil.java | 519 ----
.../sasl/SaslDataTransferClient.java | 498 ----
.../sasl/SaslDataTransferServer.java | 2 +-
.../datatransfer/sasl/SaslParticipant.java | 210 --
.../SaslResponseWithNegotiatedCipherOption.java | 33 -
...tDatanodeProtocolServerSideTranslatorPB.java | 22 +-
.../protocolPB/ClientNamenodeProtocolPB.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 135 +-
.../ClientNamenodeProtocolTranslatorPB.java | 128 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 10 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 12 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/JournalProtocolTranslatorPB.java | 2 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 3 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2321 +-----------------
.../QJournalProtocolTranslatorPB.java | 3 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 40 +-
.../hdfs/qjournal/server/JournalMetrics.java | 7 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 170 +-
.../server/balancer/BalancerParameters.java | 168 ++
.../server/blockmanagement/BlockManager.java | 97 +-
.../BlockPlacementPolicyWithNodeGroup.java | 14 +-
.../BlockStoragePolicySuite.java | 13 +-
.../server/blockmanagement/DatanodeManager.java | 4 +-
.../blockmanagement/DecommissionManager.java | 2 +-
.../hdfs/server/common/HdfsServerConstants.java | 6 -
.../hdfs/server/common/MetricsLoggerTask.java | 174 ++
.../hdfs/server/datanode/BPOfferService.java | 3 +-
.../hdfs/server/datanode/BPServiceActor.java | 7 +-
.../hdfs/server/datanode/BlockReceiver.java | 7 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 22 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 130 +-
.../hdfs/server/datanode/DataNodeMXBean.java | 9 +-
.../hdfs/server/datanode/DataXceiver.java | 11 +-
.../hdfs/server/datanode/DataXceiverServer.java | 7 +-
.../hdfs/server/datanode/FinalizedReplica.java | 15 +-
.../hdfs/server/datanode/ReplicaInfo.java | 82 -
.../server/datanode/ReplicaUnderRecovery.java | 10 -
.../datanode/ReplicaWaitingToBeRecovered.java | 15 +-
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 8 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 16 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 72 +-
.../hdfs/server/namenode/BackupImage.java | 10 +-
.../hdfs/server/namenode/CacheManager.java | 4 +-
.../hadoop/hdfs/server/namenode/DfsServlet.java | 3 +-
.../server/namenode/EditLogFileInputStream.java | 60 +-
.../server/namenode/EncryptionZoneManager.java | 4 +-
.../server/namenode/FSDirStatAndListingOp.java | 3 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 9 +-
.../hdfs/server/namenode/FSDirectory.java | 138 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 10 +
.../hdfs/server/namenode/FSEditLogLoader.java | 57 +-
.../hdfs/server/namenode/FSEditLogOp.java | 20 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 125 -
.../server/namenode/FSImageFormatPBINode.java | 9 +-
.../hdfs/server/namenode/FSNamesystem.java | 88 +-
.../server/namenode/FSPermissionChecker.java | 41 +-
.../server/namenode/FileJournalManager.java | 47 +-
.../hadoop/hdfs/server/namenode/INode.java | 14 +-
.../hdfs/server/namenode/INodeDirectory.java | 14 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 19 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 2 +-
.../hdfs/server/namenode/INodeReference.java | 10 +-
.../hdfs/server/namenode/INodeSymlink.java | 2 +-
.../hdfs/server/namenode/ImageServlet.java | 5 +-
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 192 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 6 +
.../hdfs/server/namenode/NamenodeFsck.java | 6 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 3 +
.../hadoop/hdfs/server/namenode/SafeMode.java | 3 -
.../hdfs/server/namenode/SecondaryNameNode.java | 2 +-
.../server/namenode/ha/BootstrapStandby.java | 5 +-
.../hdfs/server/namenode/ha/EditLogTailer.java | 2 +-
.../namenode/ha/IPFailoverProxyProvider.java | 4 +-
.../snapshot/DirectorySnapshottableFeature.java | 16 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +-
.../snapshot/FSImageFormatPBSnapshot.java | 6 +-
.../hdfs/server/namenode/snapshot/Snapshot.java | 3 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 42 +-
.../hdfs/tools/DFSZKFailoverController.java | 3 +-
.../org/apache/hadoop/hdfs/tools/GetGroups.java | 4 +-
.../hadoop/hdfs/tools/NNHAServiceTarget.java | 3 +-
.../apache/hadoop/hdfs/util/LongBitFormat.java | 71 -
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 +
.../services/org.apache.hadoop.fs.FileSystem | 2 -
.../src/main/resources/hdfs-default.xml | 46 +
.../src/main/webapps/hdfs/dfshealth.html | 7 +-
.../src/main/webapps/hdfs/dfshealth.js | 9 +-
.../src/main/webapps/hdfs/explorer.html | 76 +-
.../src/main/webapps/hdfs/explorer.js | 89 +-
.../src/main/webapps/static/dfs-dust.js | 12 +-
.../src/main/webapps/static/hadoop.css | 22 +
.../src/site/markdown/ArchivalStorage.md | 4 +-
.../src/site/markdown/HDFSCommands.md | 6 +-
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 6 +-
.../src/site/markdown/HdfsRollingUpgrade.md | 293 +++
.../src/site/markdown/HdfsSnapshots.md | 301 +++
.../site/resources/images/LazyPersistWrites.png | Bin 107161 -> 90224 bytes
.../src/site/xdoc/HdfsRollingUpgrade.xml | 329 ---
.../hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml | 303 ---
.../org/apache/hadoop/cli/CLITestCmdDFS.java | 8 +-
.../hadoop/cli/CLITestCmdErasureCoding.java | 5 +-
.../java/org/apache/hadoop/cli/TestAclCLI.java | 2 +-
.../apache/hadoop/cli/TestCacheAdminCLI.java | 7 +-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 6 +-
.../org/apache/hadoop/cli/TestDeleteCLI.java | 92 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 2 +-
.../java/org/apache/hadoop/cli/TestHDFSCLI.java | 4 +-
.../org/apache/hadoop/cli/TestXAttrCLI.java | 2 +-
.../hadoop/fs/permission/TestStickyBit.java | 3 +
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 3 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 24 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 2 +-
.../hadoop/hdfs/TestAppendSnapshotTruncate.java | 3 +-
.../hadoop/hdfs/TestBalancerBandwidth.java | 55 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 24 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 5 +-
.../apache/hadoop/hdfs/TestDFSPermission.java | 42 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 56 +
.../hadoop/hdfs/TestDFSShellGenericOptions.java | 13 +-
.../hadoop/hdfs/TestDefaultNameNodePort.java | 30 +-
.../hadoop/hdfs/TestEncryptedTransfer.java | 4 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 72 -
.../org/apache/hadoop/hdfs/TestFileStatus.java | 5 +-
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 7 +-
.../apache/hadoop/hdfs/TestPersistBlocks.java | 5 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 32 +
.../hdfs/TestWriteStripedFileWithFailure.java | 3 +
.../sasl/SaslDataTransferTestCase.java | 2 +-
.../datatransfer/sasl/TestSaslDataTransfer.java | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 32 +-
.../hdfs/qjournal/TestSecureNNWithQJM.java | 2 +-
.../hdfs/qjournal/server/TestJournalNode.java | 9 +
.../hdfs/server/balancer/TestBalancer.java | 199 +-
.../balancer/TestBalancerWithHANameNodes.java | 8 +-
.../TestBalancerWithMultipleNameNodes.java | 177 +-
.../balancer/TestBalancerWithNodeGroup.java | 4 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 3 -
.../TestReplicationPolicyWithNodeGroup.java | 19 +
.../hdfs/server/datanode/DataNodeTestUtils.java | 86 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../server/datanode/TestDataNodeMXBean.java | 6 +
.../datanode/TestDataNodeMetricsLogger.java | 224 ++
.../datanode/TestDataNodeRollingUpgrade.java | 8 +-
.../TestDataNodeTransferSocketSize.java | 71 +
.../datanode/TestDataNodeVolumeFailure.java | 4 +-
.../server/datanode/TestDirectoryScanner.java | 2 +-
.../datanode/extdataset/ExternalVolumeImpl.java | 2 +-
.../fsdataset/impl/FsDatasetTestUtil.java | 6 -
.../fsdataset/impl/LazyPersistTestCase.java | 16 +-
.../fsdataset/impl/TestDatanodeRestart.java | 72 -
.../datanode/fsdataset/impl/TestLazyWriter.java | 1 +
.../fsdataset/impl/TestRbwSpaceReservation.java | 452 ----
.../fsdataset/impl/TestSpaceReservation.java | 576 +++++
.../server/namenode/NNThroughputBenchmark.java | 3 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../TestCheckPointForSecurityTokens.java | 4 +-
.../namenode/TestDiskspaceQuotaUpdate.java | 9 +-
.../hdfs/server/namenode/TestEditLog.java | 103 +-
.../server/namenode/TestFSEditLogLoader.java | 13 +-
.../namenode/TestFSImageWithSnapshot.java | 3 +-
.../hdfs/server/namenode/TestFSNamesystem.java | 25 +-
.../hdfs/server/namenode/TestFileTruncate.java | 5 +-
.../hdfs/server/namenode/TestINodeFile.java | 4 +-
.../namenode/TestListCorruptFileBlocks.java | 3 +-
.../hdfs/server/namenode/TestMetaSave.java | 11 +
.../server/namenode/TestNameNodeMXBean.java | 20 +
.../namenode/TestNameNodeMetricsLogger.java | 5 +-
.../server/namenode/TestStorageRestore.java | 3 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestFailureToReadEdits.java | 3 +-
.../TestGetContentSummaryWithSnapshot.java | 126 +
.../hdfs/web/TestByteRangeInputStream.java | 79 +
.../org/apache/hadoop/tools/TestJMXGet.java | 3 +
.../test/resources/hadoop-metrics2.properties | 85 +
.../src/test/resources/log4j.properties | 13 +
.../src/test/resources/testDeleteConf.xml | 83 +
.../src/test/resources/testHDFSConf.xml | 198 +-
hadoop-mapreduce-project/CHANGES.txt | 101 +-
hadoop-mapreduce-project/bin/mapred | 8 +
.../v2/app/job/impl/TaskAttemptImpl.java | 92 +-
.../v2/app/rm/RMContainerAllocator.java | 12 +-
.../v2/app/rm/RMContainerRequestor.java | 2 +-
.../apache/hadoop/mapreduce/v2/app/MRApp.java | 11 +-
.../v2/app/job/impl/TestTaskAttempt.java | 154 ++
.../v2/app/rm/TestRMContainerAllocator.java | 113 +-
.../org/apache/hadoop/mapreduce/Cluster.java | 2 +-
.../lib/input/UncompressedSplitLineReader.java | 31 +-
.../lib/output/FileOutputCommitter.java | 52 +-
.../hadoop/mapred/TestLineRecordReader.java | 138 ++
.../lib/input/TestLineRecordReader.java | 161 ++
.../org/apache/hadoop/mapred/YARNRunner.java | 4 +
.../org/apache/hadoop/cli/CLITestCmdMR.java | 3 +-
.../org/apache/hadoop/fs/TestFileSystem.java | 14 +-
.../java/org/apache/hadoop/hdfs/NNBench.java | 14 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 10 +-
.../mapreduce/security/TestMRCredentials.java | 4 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 177 +-
.../hadoop/mapred/TestShuffleHandler.java | 129 +
hadoop-project/pom.xml | 7 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 171 ++
.../apache/hadoop/tools/HadoopArchiveLogs.java | 403 +++
.../hadoop/tools/HadoopArchiveLogsRunner.java | 180 ++
.../hadoop/tools/TestHadoopArchiveLogs.java | 293 +++
.../tools/TestHadoopArchiveLogsRunner.java | 143 ++
.../tools/mapred/UniformSizeInputFormat.java | 3 +-
.../hadoop/tools/util/ProducerConsumer.java | 19 +-
.../apache/hadoop/tools/util/WorkReport.java | 2 +-
.../apache/hadoop/tools/util/WorkRequest.java | 2 +-
.../hadoop/mapred/gridmix/SerialJobFactory.java | 2 +-
.../hadoop/yarn/sls/RumenToSLSConverter.java | 2 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 5 +
hadoop-tools/pom.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 504 ++--
.../MoveApplicationAcrossQueuesResponse.java | 5 +-
.../yarn/api/records/ApplicationReport.java | 31 +-
.../hadoop/yarn/api/records/NodeLabel.java | 16 +
.../hadoop/yarn/api/records/ResourceOption.java | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 37 +-
.../hadoop/yarn/server/api/ContainerType.java | 1 -
.../ResourceManagerAdministrationProtocol.java | 16 +-
.../RefreshNodesResourcesRequest.java | 39 +
.../RefreshNodesResourcesResponse.java | 39 +
...esourcemanager_administration_protocol.proto | 3 +-
..._server_resourcemanager_service_protos.proto | 6 +
.../src/main/proto/yarn_protos.proto | 2 +
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 12 +
.../yarn/client/api/impl/AMRMClientImpl.java | 2 +-
.../impl/ContainerManagementProtocolProxy.java | 14 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 10 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 104 +-
.../apache/hadoop/yarn/client/cli/QueueCLI.java | 11 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 61 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 53 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 31 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +-
.../application_1440536969523_0001.har/_SUCCESS | 0
.../application_1440536969523_0001.har/_index | 3 +
.../_masterindex | 2 +
.../application_1440536969523_0001.har/part-0 | Bin 0 -> 795 bytes
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +
.../impl/pb/ApplicationReportPBImpl.java | 38 +
.../org/apache/hadoop/yarn/client/RMProxy.java | 3 +-
.../apache/hadoop/yarn/client/ServerProxy.java | 3 +-
.../yarn/logaggregation/LogCLIHelpers.java | 16 +-
.../nodelabels/CommonNodeLabelsManager.java | 2 +-
.../yarn/security/ContainerTokenSelector.java | 2 +-
.../hadoop/yarn/security/NMTokenSelector.java | 2 +-
...nagerAdministrationProtocolPBClientImpl.java | 19 +
...agerAdministrationProtocolPBServiceImpl.java | 22 +
.../pb/RefreshNodesResourcesRequestPBImpl.java | 72 +
.../pb/RefreshNodesResourcesResponsePBImpl.java | 72 +
.../yarn/webapp/log/AggregatedLogsBlock.java | 7 +
.../src/main/resources/yarn-default.xml | 63 +-
.../hadoop/yarn/api/TestApplicatonReport.java | 2 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 16 +
.../hadoop/yarn/event/TestAsyncDispatcher.java | 2 +
.../logaggregation/TestAggregatedLogsBlock.java | 63 +-
.../hadoop/yarn/webapp/JerseyTestBase.java | 14 +-
.../application_1440536969523_0001.har/_SUCCESS | 0
.../application_1440536969523_0001.har/_index | 3 +
.../_masterindex | 2 +
.../application_1440536969523_0001.har/part-0 | Bin 0 -> 795 bytes
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 29 +-
.../yarn/server/api/records/NodeStatus.java | 2 +-
.../api/records/impl/pb/NodeStatusPBImpl.java | 8 +-
.../metrics/ApplicationMetricsConstants.java | 6 +
.../server/utils/YarnServerSecurityUtils.java | 142 ++
.../hadoop/yarn/server/webapp/AppBlock.java | 6 +
.../hadoop/yarn/server/webapp/WebPageUtils.java | 3 +-
.../hadoop/yarn/server/webapp/WebServices.java | 9 +
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 12 +
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../yarn/server/nodemanager/NodeManager.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 25 +-
.../amrmproxy/AMRMProxyApplicationContext.java | 70 +
.../AMRMProxyApplicationContextImpl.java | 132 +
.../nodemanager/amrmproxy/AMRMProxyService.java | 592 +++++
.../amrmproxy/AMRMProxyTokenSecretManager.java | 265 ++
.../amrmproxy/AbstractRequestInterceptor.java | 102 +
.../amrmproxy/DefaultRequestInterceptor.java | 138 ++
.../amrmproxy/RequestInterceptor.java | 71 +
.../containermanager/ContainerManagerImpl.java | 73 +-
.../localizer/LocalResourcesTrackerImpl.java | 64 +-
.../localizer/ResourceLocalizationService.java | 2 +-
.../logaggregation/AppLogAggregator.java | 2 +
.../logaggregation/AppLogAggregatorImpl.java | 6 +
.../logaggregation/LogAggregationService.java | 14 +-
.../nodelabels/AbstractNodeLabelsProvider.java | 7 +-
.../ConfigurationNodeLabelsProvider.java | 11 -
.../nodemanager/webapp/NMWebServices.java | 12 +-
.../nodemanager/webapp/dao/ContainerInfo.java | 46 +-
.../nodemanager/DummyContainerManager.java | 7 +-
.../yarn/server/nodemanager/TestEventFlow.java | 3 +-
.../nodemanager/TestNodeManagerResync.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 6 +-
.../amrmproxy/BaseAMRMProxyTest.java | 677 +++++
.../amrmproxy/MockRequestInterceptor.java | 65 +
.../amrmproxy/MockResourceManagerFacade.java | 469 ++++
.../PassThroughRequestInterceptor.java | 58 +
.../amrmproxy/TestAMRMProxyService.java | 484 ++++
.../BaseContainerManagerTest.java | 3 +-
.../containermanager/TestContainerManager.java | 5 +-
.../TestContainerManagerRecovery.java | 3 +-
.../containermanager/TestNMProxy.java | 3 +-
.../TestLocalResourcesTrackerImpl.java | 86 +-
.../localizer/TestResourceRetention.java | 2 +-
.../TestLogAggregationService.java | 17 +-
.../TestConfigurationNodeLabelsProvider.java | 69 +-
.../webapp/TestNMWebServicesContainers.java | 13 +-
.../server/resourcemanager/AdminService.java | 81 +-
.../ApplicationMasterService.java | 69 +-
.../server/resourcemanager/ClientRMService.java | 2 +-
.../resourcemanager/EmbeddedElectorService.java | 4 +
.../resourcemanager/NodesListManager.java | 144 +-
.../resourcemanager/RMActiveServiceContext.java | 16 +-
.../server/resourcemanager/RMAppManager.java | 9 +
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 12 +-
.../resourcemanager/RMFatalEventType.java | 5 +-
.../server/resourcemanager/ResourceManager.java | 4 +-
.../resourcemanager/ResourceTrackerService.java | 20 +-
.../blacklist/BlacklistManager.java | 47 +
.../blacklist/BlacklistUpdates.java | 47 +
.../blacklist/DisabledBlacklistManager.java | 45 +
.../blacklist/SimpleBlacklistManager.java | 84 +
.../metrics/ApplicationCreatedEvent.java | 16 +-
.../metrics/SystemMetricsPublisher.java | 14 +-
.../placement/PlacementManager.java | 95 +
.../placement/PlacementRule.java | 55 +
.../UserGroupMappingPlacementRule.java | 164 ++
.../resourcemanager/recovery/RMStateStore.java | 15 +-
.../resource/DynamicResourceConfiguration.java | 149 ++
.../server/resourcemanager/rmapp/RMApp.java | 6 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 59 +-
.../rmapp/attempt/RMAppAttempt.java | 7 +
.../rmapp/attempt/RMAppAttemptImpl.java | 58 +-
.../server/resourcemanager/rmnode/RMNode.java | 3 -
.../resourcemanager/rmnode/RMNodeEventType.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 456 ++--
.../scheduler/AbstractYarnScheduler.java | 2 +
.../scheduler/AppSchedulingInfo.java | 78 +-
.../scheduler/SchedulerApplicationAttempt.java | 37 +-
.../scheduler/YarnScheduler.java | 13 +
.../scheduler/capacity/CapacityScheduler.java | 153 +-
.../CapacitySchedulerConfiguration.java | 32 +-
.../scheduler/capacity/LeafQueue.java | 6 +-
.../common/fica/FiCaSchedulerUtils.java | 48 -
.../scheduler/fair/FSAppAttempt.java | 19 +-
.../scheduler/fair/FairScheduler.java | 37 +-
.../fair/FairSchedulerConfiguration.java | 22 +-
.../scheduler/fifo/FifoScheduler.java | 11 +-
.../scheduler/policy/OrderingPolicy.java | 2 +-
.../resourcemanager/webapp/AboutBlock.java | 2 +
.../webapp/CapacitySchedulerPage.java | 17 +-
.../webapp/JAXBContextResolver.java | 2 +-
.../webapp/MetricsOverviewTable.java | 3 +
.../resourcemanager/webapp/NodeLabelsPage.java | 6 +-
.../resourcemanager/webapp/RMAppBlock.java | 3 +-
.../server/resourcemanager/webapp/RMWebApp.java | 5 +
.../resourcemanager/webapp/RMWebAppFilter.java | 5 +-
.../resourcemanager/webapp/RMWebServices.java | 123 +-
.../webapp/dao/AppAttemptInfo.java | 10 +-
.../resourcemanager/webapp/dao/AppInfo.java | 29 +-
.../resourcemanager/webapp/dao/AppPriority.java | 44 +
.../dao/CapacitySchedulerLeafQueueInfo.java | 14 +-
.../resourcemanager/webapp/dao/ClusterInfo.java | 6 +
.../webapp/dao/SchedulerInfo.java | 6 +
.../yarn/server/resourcemanager/MockRM.java | 20 +-
.../server/resourcemanager/RMHATestBase.java | 2 +-
.../server/resourcemanager/TestAppManager.java | 54 +-
.../resourcemanager/TestClientRMService.java | 295 +--
.../resourcemanager/TestRMAdminService.java | 46 +
.../yarn/server/resourcemanager/TestRMHA.java | 71 +
.../resourcemanager/TestRMNodeTransitions.java | 202 +-
.../TestTokenClientRMService.java | 300 +++
.../TestWorkPreservingRMRestart.java | 21 +-
.../applicationsmanager/MockAsm.java | 10 +
.../applicationsmanager/TestAMRestart.java | 149 +-
.../blacklist/TestBlacklistManager.java | 118 +
.../TestRMAppLogAggregationStatus.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 20 +-
.../TestUserGroupMappingPlacementRule.java | 89 +
.../server/resourcemanager/rmapp/MockRMApp.java | 13 +
.../rmapp/TestNodesListManager.java | 102 +
.../rmapp/TestRMAppTransitions.java | 9 +-
.../scheduler/TestAbstractYarnScheduler.java | 14 +-
.../capacity/TestCapacityScheduler.java | 147 +-
.../scheduler/capacity/TestQueueMappings.java | 203 +-
.../scheduler/fair/FairSchedulerTestBase.java | 8 +-
.../scheduler/fair/TestFairScheduler.java | 120 +-
.../scheduler/fifo/TestFifoScheduler.java | 2 +-
.../security/TestRMDelegationTokens.java | 3 +
.../resourcemanager/webapp/TestNodesPage.java | 5 +-
.../webapp/TestRMWebServices.java | 8 +-
.../webapp/TestRMWebServicesApps.java | 34 +-
.../TestRMWebServicesAppsModification.java | 120 +
.../webapp/TestRMWebServicesCapacitySched.java | 2 +-
.../TestRMWebServicesDelegationTokens.java | 6 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 42 +-
.../src/site/markdown/CapacityScheduler.md | 6 +-
.../src/site/markdown/ResourceManagerRest.md | 152 +-
.../src/site/markdown/TimelineServer.md | 38 +-
500 files changed, 21586 insertions(+), 9191 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1080c373/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1080c373/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1080c373/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
index 0000000,23e8f57..2121dcf
mode 000000,100644..100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@@ -1,0 -1,83 +1,84 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.hadoop.hdfs.protocol;
+
+ import java.net.URI;
+
+ import org.apache.hadoop.classification.InterfaceAudience;
+ import org.apache.hadoop.classification.InterfaceStability;
+ import org.apache.hadoop.fs.FileEncryptionInfo;
+ import org.apache.hadoop.fs.LocatedFileStatus;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.permission.FsPermission;
+ import org.apache.hadoop.hdfs.DFSUtilClient;
+
+ /**
+ * Interface that represents the over the wire information
+ * including block locations for a file.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Evolving
+ public class HdfsLocatedFileStatus extends HdfsFileStatus {
+ private final LocatedBlocks locations;
+
+ /**
+ * Constructor
+ *
+ * @param length size
+ * @param isdir if this is directory
+ * @param block_replication the file's replication factor
+ * @param blocksize the file's block size
+ * @param modification_time most recent modification time
+ * @param access_time most recent access time
+ * @param permission permission
+ * @param owner owner
+ * @param group group
+ * @param symlink symbolic link
+ * @param path local path name in java UTF8 format
+ * @param fileId the file id
+ * @param locations block locations
+ * @param feInfo file encryption info
+ */
+ public HdfsLocatedFileStatus(long length, boolean isdir,
+ int block_replication, long blocksize, long modification_time,
+ long access_time, FsPermission permission, String owner, String group,
+ byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
- int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) {
++ int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
++ ErasureCodingPolicy ecPolicy) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path, fileId,
- childrenNum, feInfo, storagePolicy);
++ childrenNum, feInfo, storagePolicy, ecPolicy);
+ this.locations = locations;
+ }
+
+ public LocatedBlocks getBlockLocations() {
+ return locations;
+ }
+
+ public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
+ Path path) {
+ return new LocatedFileStatus(getLen(), isDir(), getReplication(),
+ getBlockSize(), getModificationTime(),
+ getAccessTime(),
+ getPermission(), getOwner(), getGroup(),
+ isSymlink() ? new Path(getSymlink()) : null,
+ (getFullPath(path)).makeQualified(
+ defaultUri, null), // fully-qualify path
+ DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
+ }
+ }
