Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e85174b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e85174b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e85174b Branch: refs/heads/HDFS-1312 Commit: 8e85174be583f531258bf682bda7df3cbbae728f Parents: 889dc8b ff0ee84 Author: Arpit Agarwal <[email protected]> Authored: Thu Mar 3 17:01:03 2016 -0800 Committer: Arpit Agarwal <[email protected]> Committed: Thu Mar 3 17:01:03 2016 -0800 ---------------------------------------------------------------------- dev-support/bin/dist-layout-stitching | 140 ++ dev-support/bin/dist-tar-stitching | 44 + .../main/resources/checkstyle/checkstyle.xml | 1 + hadoop-common-project/hadoop-common/CHANGES.txt | 63 +- hadoop-common-project/hadoop-common/pom.xml | 4 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 4 + .../hadoop/fs/CommonConfigurationKeys.java | 14 +- .../java/org/apache/hadoop/fs/FileSystem.java | 10 + .../org/apache/hadoop/fs/FilterFileSystem.java | 5 + .../main/java/org/apache/hadoop/io/IOUtils.java | 4 +- .../main/java/org/apache/hadoop/io/MD5Hash.java | 11 + .../hadoop/io/retry/RetryInvocationHandler.java | 22 +- .../apache/hadoop/io/retry/RetryPolicies.java | 60 +- .../metrics2/sink/RollingFileSystemSink.java | 143 +- .../apache/hadoop/security/ProviderUtils.java | 107 +- .../apache/hadoop/security/SecurityUtil.java | 42 +- .../hadoop/security/UserGroupInformation.java | 2 +- .../org/apache/hadoop/util/DataChecksum.java | 12 +- .../org/apache/hadoop/util/SysInfoWindows.java | 30 +- .../src/main/resources/core-default.xml | 140 +- .../src/main/winutils/systeminfo.c | 169 +- .../src/site/markdown/CommandsManual.md | 4 +- .../src/site/markdown/GroupsMapping.md | 191 ++ .../src/site/markdown/SecureMode.md | 2 +- .../src/site/markdown/ServiceLevelAuth.md | 2 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 6 +- .../hadoop/fs/TestChecksumFileSystem.java | 23 + .../org/apache/hadoop/fs/TestHarFileSystem.java | 2 + .../apache/hadoop/io/retry/TestRetryProxy.java | 101 +- .../org/apache/hadoop/ipc/RPCCallBenchmark.java | 4 +- .../hadoop/ipc/TestMultipleProtocolServer.java | 14 +- .../org/apache/hadoop/ipc/TestProtoBufRpc.java | 137 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 1013 +++++------ .../hadoop/ipc/TestRPCServerShutdown.java | 106 ++ .../java/org/apache/hadoop/ipc/TestRpcBase.java | 295 ++++ .../sink/RollingFileSystemSinkTestBase.java | 7 +- .../sink/TestRollingFileSystemSink.java | 10 +- .../hadoop/service/TestCompositeService.java | 480 ++++- .../apache/hadoop/test/GenericTestUtils.java | 56 + .../apache/hadoop/util/TestSysInfoWindows.java | 19 +- .../hadoop-common/src/test/proto/test.proto | 33 + .../src/test/proto/test_rpc_service.proto | 7 + hadoop-dist/pom.xml | 148 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 255 +-- .../org/apache/hadoop/hdfs/DFSInputStream.java | 72 +- .../hadoop/hdfs/DFSStripedInputStream.java | 49 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 85 + .../hadoop/hdfs/DistributedFileSystem.java | 27 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 416 +++++ .../hdfs/client/HdfsClientConfigKeys.java | 7 + .../hadoop/hdfs/protocol/ClientProtocol.java | 13 + .../protocol/datatransfer/IOStreamPair.java | 11 +- .../ClientNamenodeProtocolTranslatorPB.java | 12 + .../hadoop/hdfs/web/URLConnectionFactory.java | 30 +- .../src/main/proto/ClientNamenodeProtocol.proto | 9 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 65 + .../bkjournal/TestBookKeeperAsHASharedDir.java | 46 +- .../src/test/resources/log4j.properties | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 3 - ...tNamenodeProtocolServerSideTranslatorPB.java | 17 + .../hadoop/hdfs/server/balancer/Balancer.java | 22 +- .../server/blockmanagement/BlockManager.java | 139 +- .../blockmanagement/BlockPlacementPolicy.java | 48 +- .../BlockPlacementPolicyDefault.java | 20 +- .../blockmanagement/ErasureCodingWork.java | 6 + .../blockmanagement/ExcessReplicaMap.java | 106 ++ .../hdfs/server/datanode/BPOfferService.java | 17 +- .../hdfs/server/datanode/BPServiceActor.java | 19 +- .../server/datanode/BlockChecksumHelper.java | 254 +++ .../hdfs/server/datanode/BlockReceiver.java | 8 +- .../hdfs/server/datanode/BlockSender.java | 1 - .../hadoop/hdfs/server/datanode/DNConf.java | 4 + .../hadoop/hdfs/server/datanode/DataNode.java | 52 +- .../server/datanode/DataNodeLayoutVersion.java | 5 +- .../hdfs/server/datanode/DataStorage.java | 11 +- .../hdfs/server/datanode/DataXceiver.java | 176 +- .../hdfs/server/datanode/DatanodeUtil.java | 4 +- .../datanode/IncrementalBlockReportManager.java | 51 +- .../erasurecode/ErasureCodingWorker.java | 50 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 4 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 12 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 + .../hdfs/server/namenode/FSDirAttrOp.java | 74 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 3 +- .../hdfs/server/namenode/FSDirectory.java | 6 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 109 +- .../hdfs/server/namenode/FSEditLogAsync.java | 322 ++++ .../hdfs/server/namenode/FSEditLogOp.java | 213 +-- .../hdfs/server/namenode/FSEditLogOpCodes.java | 108 +- .../hadoop/hdfs/server/namenode/FSImage.java | 2 +- .../server/namenode/FSImageFormatPBINode.java | 12 +- .../hdfs/server/namenode/FSNamesystem.java | 22 + .../hadoop/hdfs/server/namenode/NameNode.java | 1 - .../hdfs/server/namenode/NameNodeRpcServer.java | 7 + .../hdfs/server/namenode/NamenodeFsck.java | 6 +- .../namenode/metrics/NameNodeMetrics.java | 4 +- .../protocol/StorageReceivedDeletedBlocks.java | 7 + .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 48 +- .../OfflineImageReconstructor.java | 1639 ++++++++++++++++++ .../OfflineImageViewerPB.java | 22 +- .../offlineImageViewer/PBImageXmlWriter.java | 248 ++- .../src/main/resources/hdfs-default.xml | 135 +- .../src/site/markdown/ArchivalStorage.md | 15 + .../src/site/markdown/HdfsPermissionsGuide.md | 8 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 28 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 3 + .../hadoop/hdfs/TestApplyingStoragePolicy.java | 268 +++ .../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +- .../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 29 +- .../hadoop/hdfs/TestDatanodeRegistration.java | 6 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 8 + .../hadoop/hdfs/TestReconstructStripedFile.java | 4 +- .../hdfs/server/balancer/TestBalancer.java | 125 +- .../server/blockmanagement/TestNodeCount.java | 7 +- .../TestOverReplicatedBlocks.java | 10 +- ...constructStripedBlocksWithRackAwareness.java | 133 +- .../blockmanagement/TestReplicationPolicy.java | 15 +- .../TestReplicationPolicyWithNodeGroup.java | 2 +- .../TestReplicationPolicyWithUpgradeDomain.java | 12 +- .../server/datanode/SimulatedFSDataset.java | 11 +- .../server/datanode/TestBPOfferService.java | 2 +- .../hdfs/server/datanode/TestBatchIbr.java | 263 +++ .../server/datanode/TestDataNodeMetrics.java | 48 +- .../TestDataXceiverLazyPersistHint.java | 6 +- .../datanode/TestIncrementalBlockReports.java | 2 +- .../extdataset/ExternalDatasetImpl.java | 4 +- .../impl/TestInterDatanodeProtocol.java | 6 +- .../hdfs/server/namenode/TestAuditLogs.java | 15 +- .../hdfs/server/namenode/TestEditLog.java | 59 +- .../server/namenode/TestEditLogAutoroll.java | 26 + .../namenode/TestEditLogJournalFailures.java | 35 +- .../hdfs/server/namenode/TestEditLogRace.java | 144 +- .../server/namenode/TestFSEditLogLoader.java | 37 +- .../server/namenode/TestNameNodeRecovery.java | 31 +- .../server/namenode/ha/TestEditLogTailer.java | 39 +- .../namenode/metrics/TestNameNodeMetrics.java | 2 +- .../hdfs/tools/TestStoragePolicyCommands.java | 63 + .../TestOfflineImageViewer.java | 56 +- .../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 37 +- .../sink/TestRollingFileSystemSinkWithHdfs.java | 34 +- ...TestRollingFileSystemSinkWithSecureHdfs.java | 3 +- .../apache/hadoop/security/TestPermission.java | 13 + .../resources/hadoop-56-layout-datanode-dir.tgz | Bin 0 -> 198996 bytes .../resources/hadoop-to-57-dn-layout-dir.txt | 24 + hadoop-mapreduce-project/CHANGES.txt | 5 + .../hadoop/mapreduce/v2/app/TestMRApp.java | 104 +- .../mapreduce/v2/jobhistory/JHAdminConfig.java | 8 +- .../src/main/resources/mapred-default.xml | 27 +- .../mapreduce/v2/hs/CachedHistoryStorage.java | 146 +- .../v2/hs/webapp/dao/AMAttemptInfo.java | 4 +- .../hadoop/mapreduce/v2/hs/TestJobHistory.java | 255 ++- .../v2/hs/webapp/TestHsWebServicesJobs.java | 11 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 47 +- .../hadoop/fs/s3a/TestS3AConfiguration.java | 34 + .../hadoop/fs/azure/SimpleKeyProvider.java | 5 +- .../hadoop-azure/src/site/markdown/index.md | 6 +- .../fs/azure/TestWasbUriAndConfiguration.java | 35 + .../src/site/markdown/DistCp.md.vm | 13 +- hadoop-tools/hadoop-openstack/pom.xml | 4 +- hadoop-yarn-project/CHANGES.txt | 33 + .../hadoop/yarn/conf/YarnConfiguration.java | 7 + .../yarn/webapp/log/AggregatedLogsBlock.java | 27 + .../src/main/resources/yarn-default.xml | 10 + .../logaggregation/TestAggregatedLogsBlock.java | 20 + ...pplicationHistoryManagerOnTimelineStore.java | 9 + .../nodemanager/DefaultContainerExecutor.java | 32 +- .../logaggregation/AppLogAggregatorImpl.java | 68 +- .../impl/container-executor.c | 54 +- .../test/test-container-executor.c | 99 +- .../TestDefaultContainerExecutor.java | 21 +- .../server/nodemanager/TestNMAuditLogger.java | 40 +- .../TestLogAggregationService.java | 59 + .../resourcemanager/NodesListManager.java | 13 +- .../resourcemanager/rmnode/RMNodeImpl.java | 4 + .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../scheduler/SchedulerNode.java | 172 +- .../scheduler/SchedulerNodeReport.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 18 +- .../scheduler/capacity/ParentQueue.java | 2 +- .../allocator/IncreaseContainerAllocator.java | 4 +- .../allocator/RegularContainerAllocator.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 6 +- .../scheduler/fair/FairScheduler.java | 8 +- .../scheduler/fifo/FifoScheduler.java | 20 +- .../resourcemanager/TestRMAuditLogger.java | 44 +- .../resourcemanager/TestRMNodeTransitions.java | 14 + .../resourcemanager/TestResourceManager.java | 7 + .../TestWorkPreservingRMRestart.java | 12 +- .../scheduler/TestAbstractYarnScheduler.java | 4 +- .../capacity/TestCapacityScheduler.java | 39 +- .../scheduler/capacity/TestChildQueueOrder.java | 4 +- .../capacity/TestContainerAllocation.java | 4 +- .../capacity/TestContainerResizing.java | 2 +- .../capacity/TestIncreaseAllocationExpirer.java | 2 +- .../scheduler/capacity/TestLeafQueue.java | 16 +- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/capacity/TestReservations.java | 196 +-- .../scheduler/capacity/TestUtils.java | 2 +- .../scheduler/fair/TestFSAppAttempt.java | 4 +- .../scheduler/fair/TestFairScheduler.java | 4 +- .../scheduler/fifo/TestFifoScheduler.java | 6 +- .../TestMiniYarnClusterNodeUtilization.java | 6 +- 204 files changed, 9587 insertions(+), 2682 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e85174b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e85174b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ----------------------------------------------------------------------
