Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bb198e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bb198e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bb198e7 Branch: refs/heads/YARN-2928 Commit: 6bb198e7298f45c5260b4dd150fad09c48bb7629 Parents: 6b4dc96 1a68fc4 Author: Zhijie Shen <[email protected]> Authored: Wed Feb 25 09:20:02 2015 -0800 Committer: Zhijie Shen <[email protected]> Committed: Wed Feb 25 09:20:02 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 43 ++- .../apache/hadoop/fs/AbstractFileSystem.java | 7 +- .../java/org/apache/hadoop/fs/StorageType.java | 93 +++++ .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 10 + .../org/apache/hadoop/fs/shell/find/Find.java | 29 +- .../main/java/org/apache/hadoop/ipc/Client.java | 3 +- .../metrics2/impl/MetricsSinkAdapter.java | 15 +- .../hadoop/net/unix/DomainSocketWatcher.java | 45 ++- .../hadoop/util/NodeHealthScriptRunner.java | 345 ++++++++++++++++++ .../apache/hadoop/util/curator/ChildReaper.java | 234 ++++++++++++ .../src/main/resources/core-default.xml | 5 +- .../src/site/markdown/CommandsManual.md | 15 +- .../hadoop-common/src/site/markdown/Metrics.md | 7 +- .../src/site/markdown/ServiceLevelAuth.md | 2 +- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 6 - .../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 33 ++ .../apache/hadoop/ha/ClientBaseWithFixes.java | 2 +- .../metrics2/impl/TestMetricsSystemImpl.java | 50 +++ .../net/unix/TestDomainSocketWatcher.java | 65 +++- .../hadoop/util/TestNodeHealthScriptRunner.java | 136 +++++++ .../hadoop/util/curator/TestChildReaper.java | 208 +++++++++++ .../hadoop/fs/http/client/HttpFSFileSystem.java | 24 +- .../hadoop/fs/http/server/FSOperations.java | 43 ++- .../http/server/HttpFSParametersProvider.java | 20 ++ .../hadoop/fs/http/server/HttpFSServer.java | 10 + .../fs/http/client/BaseTestHttpFSWith.java | 40 ++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 44 +++ .../hadoop-hdfs/src/CMakeLists.txt | 8 + .../apache/hadoop/hdfs/BlockReaderFactory.java | 1 + .../apache/hadoop/hdfs/BlockReaderLocal.java | 1 + .../hadoop/hdfs/BlockReaderLocalLegacy.java | 1 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 1 + .../org/apache/hadoop/hdfs/DFSInputStream.java | 1 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 3 +- .../hadoop/hdfs/DistributedFileSystem.java | 1 + .../org/apache/hadoop/hdfs/StorageType.java | 93 ----- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 2 +- .../hdfs/protocol/BlockStoragePolicy.java | 2 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hdfs/protocol/DatanodeInfoWithStorage.java | 2 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 2 +- .../QuotaByStorageTypeExceededException.java | 2 +- .../datatransfer/DataTransferProtocol.java | 2 +- .../hdfs/protocol/datatransfer/Sender.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 2 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2 +- .../hadoop/hdfs/server/balancer/Balancer.java | 2 +- .../hdfs/server/balancer/BalancingPolicy.java | 2 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 2 +- .../server/blockmanagement/BlockManager.java | 2 +- .../blockmanagement/BlockPlacementPolicy.java | 2 +- .../BlockPlacementPolicyDefault.java | 2 +- .../BlockPlacementPolicyWithNodeGroup.java | 3 +- .../BlockStoragePolicySuite.java | 2 +- .../blockmanagement/DatanodeDescriptor.java | 2 +- .../blockmanagement/DatanodeStorageInfo.java | 2 +- .../hdfs/server/datanode/BPOfferService.java | 2 +- .../hdfs/server/datanode/BlockReceiver.java | 3 +- .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../hdfs/server/datanode/DataXceiver.java | 28 +- .../server/datanode/ReportBadBlockAction.java | 2 +- .../hdfs/server/datanode/StorageLocation.java | 2 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 2 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 2 +- .../RoundRobinVolumeChoosingPolicy.java | 1 - .../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +- .../datanode/fsdataset/impl/FsVolumeList.java | 2 +- .../datanode/metrics/DataNodeMetrics.java | 36 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 6 +- .../namenode/DirectoryWithQuotaFeature.java | 2 +- .../hdfs/server/namenode/FSDirAttrOp.java | 2 +- .../hdfs/server/namenode/FSDirConcatOp.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 2 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +- .../hdfs/server/namenode/FSEditLogOp.java | 2 +- .../hadoop/hdfs/server/namenode/FSImage.java | 2 +- .../server/namenode/FSImageFormatPBINode.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../hdfs/server/namenode/INodeDirectory.java | 4 +- .../namenode/INodeDirectoryAttributes.java | 2 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 5 +- .../server/namenode/MetaRecoveryContext.java | 6 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 2 +- .../namenode/QuotaByStorageTypeEntry.java | 4 +- .../hdfs/server/namenode/QuotaCounts.java | 2 +- .../namenode/metrics/NameNodeMetrics.java | 25 ++ .../snapshot/FSImageFormatPBSnapshot.java | 2 +- .../server/namenode/snapshot/FileDiffList.java | 2 +- .../snapshot/FileWithSnapshotFeature.java | 2 +- .../hdfs/server/protocol/BlockCommand.java | 2 +- .../server/protocol/BlocksWithLocations.java | 2 +- .../hdfs/server/protocol/DatanodeStorage.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 59 +-- .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 20 -- .../hdfs/tools/DFSZKFailoverController.java | 9 +- .../apache/hadoop/hdfs/util/LongBitFormat.java | 4 + .../src/main/native/libhdfs/common/htable.c | 24 +- .../src/main/native/libhdfs/test/test_htable.c | 100 ++++++ .../src/main/resources/hdfs-default.xml | 15 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 11 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 1 + .../hdfs/MiniDFSClusterWithNodeGroup.java | 4 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 1 + .../hadoop/hdfs/TestDataTransferProtocol.java | 1 + .../hdfs/TestWriteBlockGetsBlockLengthHint.java | 1 + .../hadoop/hdfs/protocolPB/TestPBHelper.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 6 +- .../blockmanagement/TestBlockManager.java | 2 +- .../blockmanagement/TestDatanodeManager.java | 2 +- .../blockmanagement/TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyWithNodeGroup.java | 2 +- .../server/datanode/BlockReportTestBase.java | 3 +- .../server/datanode/SimulatedFSDataset.java | 2 +- .../hdfs/server/datanode/TestBlockRecovery.java | 2 +- .../server/datanode/TestBlockReplacement.java | 2 +- .../hdfs/server/datanode/TestDataDirs.java | 2 +- .../server/datanode/TestDataNodeMetrics.java | 45 +++ .../TestDatanodeProtocolRetryPolicy.java | 233 ++++++++++++ .../server/datanode/TestDirectoryScanner.java | 7 +- .../hdfs/server/datanode/TestDiskError.java | 2 +- .../server/datanode/TestSimulatedFSDataset.java | 2 +- .../hdfs/server/datanode/TestStorageReport.java | 5 +- .../extdataset/ExternalDatasetImpl.java | 2 +- .../datanode/extdataset/ExternalVolumeImpl.java | 2 +- .../fsdataset/impl/LazyPersistTestCase.java | 6 +- .../fsdataset/impl/TestFsDatasetImpl.java | 2 +- .../fsdataset/impl/TestFsVolumeList.java | 2 +- .../fsdataset/impl/TestLazyPersistFiles.java | 5 +- .../fsdataset/impl/TestScrLazyPersistFiles.java | 6 +- .../fsdataset/impl/TestWriteToReplica.java | 2 +- .../hadoop/hdfs/server/mover/TestMover.java | 8 +- .../hdfs/server/mover/TestStorageMover.java | 2 +- .../hdfs/server/namenode/TestAddBlockRetry.java | 1 - .../hdfs/server/namenode/TestFSImage.java | 48 +++ .../hdfs/server/namenode/TestFileTruncate.java | 221 ++++++++++++ .../server/namenode/TestQuotaByStorageType.java | 9 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 2 +- .../namenode/metrics/TestNameNodeMetrics.java | 20 ++ .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 20 -- .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 4 +- .../resources/image-with-zero-block-size.tar.gz | Bin 0 -> 1452 bytes .../src/test/resources/testHDFSConf.xml | 4 +- hadoop-mapreduce-project/CHANGES.txt | 12 +- .../mapreduce/v2/app/webapp/TestAMWebApp.java | 2 +- .../apache/hadoop/mapred/JobEndNotifier.java | 27 +- .../mapred/lib/MultithreadedMapRunner.java | 8 +- .../hadoop/mapred/ResourceMgrDelegate.java | 12 + .../apache/hadoop/fs/slive/ArgumentParser.java | 2 + .../apache/hadoop/fs/slive/ConfigExtractor.java | 59 +++ .../apache/hadoop/fs/slive/ConfigMerger.java | 35 ++ .../apache/hadoop/fs/slive/ConfigOption.java | 9 + .../org/apache/hadoop/fs/slive/Constants.java | 4 +- .../hadoop/fs/slive/OperationFactory.java | 3 + .../org/apache/hadoop/fs/slive/TestSlive.java | 27 ++ .../org/apache/hadoop/fs/slive/TruncateOp.java | 114 ++++++ .../hadoop/mapred/TestClientRedirect.java | 8 + .../apache/hadoop/maven/plugin/util/Exec.java | 7 +- hadoop-project/pom.xml | 1 - .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 5 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 51 ++- .../src/site/markdown/tools/hadoop-aws/index.md | 4 +- .../apache/hadoop/fs/s3a/TestS3ABlocksize.java | 93 +++++ .../src/test/resources/log4j.properties | 18 + hadoop-yarn-project/CHANGES.txt | 44 +++ .../yarn/api/ApplicationClientProtocol.java | 18 + .../GetLabelsToNodesRequest.java | 41 +++ .../GetLabelsToNodesResponse.java | 45 +++ .../main/proto/applicationclient_protocol.proto | 1 + .../src/main/proto/yarn_protos.proto | 5 + .../src/main/proto/yarn_service_protos.proto | 8 + .../hadoop/yarn/client/api/YarnClient.java | 31 ++ .../yarn/client/api/impl/YarnClientImpl.java | 15 + .../apache/hadoop/yarn/client/cli/LogsCLI.java | 11 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 + .../yarn/client/api/impl/TestYarnClient.java | 75 +++- .../hadoop/yarn/client/cli/TestLogsCLI.java | 1 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 1 + .../ApplicationClientProtocolPBClientImpl.java | 19 + .../ApplicationClientProtocolPBServiceImpl.java | 21 ++ .../impl/pb/GetLabelsToNodesRequestPBImpl.java | 121 +++++++ .../impl/pb/GetLabelsToNodesResponsePBImpl.java | 184 ++++++++++ .../apache/hadoop/yarn/client/ServerProxy.java | 1 - .../hadoop/yarn/api/TestPBImplRecords.java | 12 + .../security/http/RMAuthenticationFilter.java | 2 - .../nodemanager/NodeHealthCheckerService.java | 12 +- .../nodemanager/NodeHealthScriptRunner.java | 356 ------------------- .../yarn/server/nodemanager/NodeManager.java | 26 +- .../logaggregation/AppLogAggregatorImpl.java | 4 +- .../yarn/server/nodemanager/TestEventFlow.java | 7 +- .../nodemanager/TestNodeHealthService.java | 86 ++--- .../BaseContainerManagerTest.java | 7 +- .../TestLogAggregationService.java | 25 +- .../webapp/TestContainerLogsPage.java | 13 +- .../nodemanager/webapp/TestNMWebServer.java | 13 +- .../nodemanager/webapp/TestNMWebServices.java | 7 +- .../webapp/TestNMWebServicesApps.java | 6 +- .../webapp/TestNMWebServicesContainers.java | 5 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 8 +- .../server/resourcemanager/ClientRMService.java | 15 + .../resourcemanager/ResourceTrackerService.java | 9 +- .../rmapp/attempt/RMAppAttemptImpl.java | 2 + .../resourcemanager/rmnode/RMNodeImpl.java | 111 +++--- .../rmnode/RMNodeReconnectEvent.java | 9 +- .../scheduler/fair/FSAppAttempt.java | 6 +- .../server/resourcemanager/webapp/AppBlock.java | 33 +- .../resourcemanager/webapp/AppsBlock.java | 6 +- .../resourcemanager/webapp/dao/AppInfo.java | 8 +- .../org/apache/hadoop/test/YarnTestDriver.java | 60 ++++ .../ParameterizedSchedulerTestBase.java | 12 + .../resourcemanager/TestApplicationCleanup.java | 121 +++++++ .../resourcemanager/TestClientRMService.java | 75 ++++ .../resourcemanager/TestRMNodeTransitions.java | 4 +- .../TestWorkPreservingRMRestart.java | 79 +--- .../recovery/RMStateStoreTestBase.java | 19 +- .../recovery/TestZKRMStateStorePerf.java | 277 +++++++++++++++ .../attempt/TestRMAppAttemptTransitions.java | 30 ++ .../scheduler/capacity/TestQueueMappings.java | 2 + .../server/webproxy/WebAppProxyServlet.java | 21 +- .../server/webproxy/TestWebAppProxyServlet.java | 23 +- 220 files changed, 4458 insertions(+), 994 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb198e7/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb198e7/hadoop-yarn-project/CHANGES.txt ----------------------------------------------------------------------
