Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/566c8b74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/566c8b74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/566c8b74 Branch: refs/heads/HADOOP-13345 Commit: 566c8b74fe7a5a72fc2cc8c8e5a449b0070d6b7c Parents: 090924a 7992426 Author: Steve Loughran <[email protected]> Authored: Mon Mar 13 13:55:38 2017 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Mar 13 13:55:38 2017 +0000 ---------------------------------------------------------------------- hadoop-client-modules/hadoop-client/pom.xml | 4 - .../hadoop-cloud-storage/pom.xml | 4 - hadoop-common-project/hadoop-common/pom.xml | 5 - .../src/main/conf/log4j.properties | 9 + .../fs/CommonConfigurationKeysPublic.java | 29 +- .../org/apache/hadoop/fs/shell/Command.java | 14 +- .../java/org/apache/hadoop/io/SequenceFile.java | 20 +- .../main/java/org/apache/hadoop/ipc/Client.java | 4 +- .../java/org/apache/hadoop/net/InnerNode.java | 2 +- .../org/apache/hadoop/net/InnerNodeImpl.java | 22 +- .../org/apache/hadoop/net/NetworkTopology.java | 27 +- .../java/org/apache/hadoop/util/ConfTest.java | 2 +- .../org/apache/hadoop/util/StringUtils.java | 11 + .../src/main/resources/core-default.xml | 35 ++ .../src/site/markdown/CLIMiniCluster.md.vm | 1 + .../src/site/markdown/ClusterSetup.md | 4 +- .../hadoop-common/src/site/markdown/Metrics.md | 7 +- .../site/markdown/filesystem/introduction.md | 85 ++++ .../conf/TestCommonConfigurationFields.java | 1 + .../org/apache/hadoop/io/TestSequenceFile.java | 68 ++- .../apache/hadoop/net/TestClusterTopology.java | 5 +- hadoop-common-project/hadoop-kms/pom.xml | 4 - .../key/kms/server/KMSAuthenticationFilter.java | 3 +- .../hadoop-kms/src/site/markdown/index.md.vm | 2 +- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 1 + .../apache/hadoop/oncrpc/SimpleUdpServer.java | 1 + .../java/org/apache/hadoop/portmap/Portmap.java | 2 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../hadoop/hdfs/DistributedFileSystem.java | 11 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 5 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 52 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 10 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 6 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 2 + .../hdfs/web/resources/DeleteOpParam.java | 11 +- .../hadoop/hdfs/web/resources/GetOpParam.java | 11 +- .../hadoop/hdfs/web/resources/PostOpParam.java | 11 +- .../hadoop/hdfs/web/resources/PutOpParam.java | 11 +- .../src/main/proto/erasurecoding.proto | 2 +- .../src/main/proto/hdfs.proto | 2 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 - .../dev-support/findbugsExcludeFile.xml | 6 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 4 +- .../server/blockmanagement/BlockManager.java | 22 +- .../blockmanagement/DatanodeDescriptor.java | 13 + .../server/blockmanagement/SlowPeerTracker.java | 4 +- .../apache/hadoop/hdfs/server/common/Util.java | 17 + .../hdfs/server/datanode/BPServiceActor.java | 2 +- .../hadoop/hdfs/server/datanode/DNConf.java | 17 +- .../hadoop/hdfs/server/datanode/DataNode.java | 83 +++- .../hdfs/server/datanode/FileIoProvider.java | 2 +- .../server/datanode/ProfilingFileIoEvents.java | 11 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 28 +- .../datanode/fsdataset/impl/FsVolumeList.java | 13 +- .../datanode/metrics/DataNodeDiskMetrics.java | 181 +++++++ .../datanode/metrics/DataNodePeerMetrics.java | 6 +- .../datanode/metrics/OutlierDetector.java | 182 +++++++ .../datanode/metrics/SlowNodeDetector.java | 194 -------- .../web/RestCsrfPreventionFilterHandler.java | 6 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 33 +- .../namenode/ErasureCodingPolicyManager.java | 107 ++-- .../server/namenode/FSDirErasureCodingOp.java | 139 +++--- .../server/namenode/FSDirStatAndListingOp.java | 4 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 48 +- .../hdfs/server/namenode/FSDirectory.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 23 +- .../hdfs/server/namenode/FSImageFormat.java | 6 +- .../server/namenode/FSImageFormatPBINode.java | 29 +- .../hdfs/server/namenode/FSNamesystem.java | 28 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 79 ++- .../server/namenode/INodeFileAttributes.java | 6 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 +- .../hdfs/server/namenode/NamenodeFsck.java | 6 +- .../snapshot/FSImageFormatPBSnapshot.java | 9 +- .../apache/hadoop/hdfs/tools/CacheAdmin.java | 8 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 18 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 57 ++- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 50 +- .../tools/offlineImageViewer/FSImageLoader.java | 7 +- .../OfflineImageReconstructor.java | 4 + .../offlineImageViewer/PBImageXmlWriter.java | 19 +- .../hdfs/tools/snapshot/SnapshotDiff.java | 19 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 + .../hadoop-hdfs/src/main/proto/fsimage.proto | 1 + .../src/main/resources/hdfs-default.xml | 13 +- .../src/main/webapps/hdfs/dfshealth.html | 2 + .../src/main/webapps/hdfs/dfshealth.js | 1 + .../src/site/markdown/HDFSCommands.md | 6 +- .../src/site/markdown/HDFSErasureCoding.md | 45 +- .../apache/hadoop/cli/TestErasureCodingCLI.java | 3 + .../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 ++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 17 +- .../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 ++++++++++ .../hdfs/ErasureCodeBenchmarkThroughput.java | 8 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 + .../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 + .../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +- .../hadoop/hdfs/TestDFSNetworkTopology.java | 260 ++++++++++ .../TestDFSRSDefault10x4StripedInputStream.java | 2 +- ...TestDFSRSDefault10x4StripedOutputStream.java | 2 +- ...fault10x4StripedOutputStreamWithFailure.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 8 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 7 +- .../TestDFSStripedOutputStreamWithFailure.java | 6 +- .../hdfs/TestDFSXORStripedInputStream.java | 2 +- .../hdfs/TestDFSXORStripedOutputStream.java | 2 +- ...estDFSXORStripedOutputStreamWithFailure.java | 2 +- .../apache/hadoop/hdfs/TestDecommission.java | 48 ++ .../hdfs/TestDecommissionWithStriped.java | 6 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 234 +++++++-- .../TestErasureCodingPolicyWithSnapshot.java | 13 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 6 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 5 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 5 +- .../hadoop/hdfs/TestMaintenanceState.java | 162 ++++++ .../java/org/apache/hadoop/hdfs/TestQuota.java | 26 + .../hdfs/TestReadStripedFileWithDecoding.java | 6 +- .../TestReadStripedFileWithMissingBlocks.java | 5 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 8 +- .../hdfs/TestSafeModeWithStripedFile.java | 6 +- .../hadoop/hdfs/TestSnapshotCommands.java | 34 ++ .../TestUnsetAndChangeDirectoryEcPolicy.java | 26 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 6 +- .../hdfs/TestWriteStripedFileWithFailure.java | 6 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 12 +- .../hdfs/server/balancer/TestBalancer.java | 6 +- .../blockmanagement/TestBlockInfoStriped.java | 4 +- .../TestBlockTokenWithDFSStriped.java | 8 +- .../TestLowRedundancyBlockQueues.java | 4 +- ...constructStripedBlocksWithRackAwareness.java | 13 +- .../TestSequentialBlockGroupId.java | 8 +- .../TestSortLocatedStripedBlock.java | 4 +- .../hdfs/server/datanode/TestBlockRecovery.java | 3 +- .../TestDataNodeErasureCodingMetrics.java | 6 +- .../TestDataNodeVolumeFailureReporting.java | 116 ++++- .../datanode/TestDataNodeVolumeMetrics.java | 4 +- .../TestDataNodeOutlierDetectionViaMetrics.java | 6 +- .../datanode/metrics/TestSlowNodeDetector.java | 30 +- .../hadoop/hdfs/server/mover/TestMover.java | 6 +- .../TestAddOverReplicatedStripedBlocks.java | 7 +- .../namenode/TestAddStripedBlockInFBR.java | 6 +- .../server/namenode/TestAddStripedBlocks.java | 6 +- .../server/namenode/TestEnabledECPolicies.java | 151 ++++++ .../server/namenode/TestFSEditLogLoader.java | 7 +- .../hdfs/server/namenode/TestFSImage.java | 39 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 37 +- .../hdfs/server/namenode/TestINodeFile.java | 64 ++- .../server/namenode/TestNameNodeMXBean.java | 14 +- .../namenode/TestQuotaWithStripedBlocks.java | 6 +- .../namenode/TestReconstructStripedBlocks.java | 8 +- .../server/namenode/TestStripedINodeFile.java | 82 ++- .../shortcircuit/TestShortCircuitLocalRead.java | 2 +- .../TestOfflineImageViewer.java | 159 +++++- ...TestOfflineImageViewerWithStripedBlocks.java | 9 +- .../hadoop/hdfs/web/resources/TestParam.java | 41 ++ .../apache/hadoop/net/TestNetworkTopology.java | 6 +- .../hadoop/tools/TestHdfsConfigFields.java | 2 - .../test/resources/testErasureCodingConf.xml | 21 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 12 +- .../java/org/apache/hadoop/mapreduce/Job.java | 2 +- .../ClientDistributedCacheManager.java | 17 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 12 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +- .../mapreduce/MiniHadoopClusterManager.java | 7 +- .../mapreduce/security/CredentialsTestJob.java | 5 +- .../org/apache/hadoop/examples/pi/package.html | 46 +- hadoop-project/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 7 +- .../services/org.apache.hadoop.fs.FileSystem | 1 - .../hadoop/fs/s3native/TestS3Credentials.java | 18 - .../META-INF/org.apache.hadoop.fs.FileSystem | 16 - .../src/site/markdown/index.md | 55 +-- .../fs/azure/AzureNativeFileSystemStore.java | 5 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 136 ++++- .../fs/azure/RemoteWasbAuthorizerImpl.java | 190 +++++++ .../fs/azure/WasbAuthorizationException.java | 40 ++ .../fs/azure/WasbAuthorizationOperations.java | 44 ++ .../fs/azure/WasbAuthorizerInterface.java | 53 ++ .../hadoop/fs/azure/WasbRemoteCallHelper.java | 71 ++- .../hadoop-azure/src/site/markdown/index.md | 34 ++ .../fs/azure/AzureBlobStorageTestAccount.java | 61 +-- .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 102 ++++ .../TestNativeAzureFileSystemAuthorization.java | 344 +++++++++++++ .../fs/azure/TestWasbRemoteCallHelper.java | 344 +++++++++++++ .../src/test/resources/azure-test.xml | 28 +- .../src/test/resources/log4j.properties | 22 + .../services/org.apache.hadoop.fs.FileSystem | 16 - hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 8 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 8 - .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 2 + .../java/org/apache/hadoop/record/package.html | 74 +-- .../org/apache/hadoop/typedbytes/package.html | 13 +- hadoop-tools/hadoop-tools-dist/pom.xml | 6 + .../hadoop/yarn/conf/YarnConfiguration.java | 20 + .../yarn/client/api/impl/TestAMRMClient.java | 157 +++--- .../logaggregation/AggregatedLogFormat.java | 52 +- .../hadoop/yarn/util/resource/Resources.java | 7 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 43 +- .../src/main/resources/yarn-default.xml | 37 ++ .../hadoop/yarn/conf/TestYarnConfiguration.java | 16 + .../yarn/util/resource/TestResources.java | 28 +- .../apache/hadoop/yarn/webapp/TestWebApp.java | 31 +- .../TestFileSystemApplicationHistoryStore.java | 14 +- .../server/scheduler/SchedulerRequestKey.java | 12 +- .../server/nodemanager/ContainerExecutor.java | 9 + .../nodemanager/LinuxContainerExecutor.java | 23 + .../launcher/ContainerLaunch.java | 34 +- .../DelegatingLinuxContainerRuntime.java | 30 +- .../JavaSandboxLinuxContainerRuntime.java | 495 +++++++++++++++++++ .../runtime/LinuxContainerRuntimeConstants.java | 2 + .../linux/runtime/docker/DockerPullCommand.java | 31 ++ .../executor/ContainerPrepareContext.java | 119 +++++ .../src/main/resources/java.policy | 63 +++ .../TestJavaSandboxLinuxContainerRuntime.java | 364 ++++++++++++++ .../runtime/docker/TestDockerPullCommand.java | 49 ++ .../TestAppLogAggregatorImpl.java | 8 +- .../server/resourcemanager/AdminService.java | 9 + .../server/resourcemanager/ClientRMService.java | 15 +- ...MCriticalThreadUncaughtExceptionHandler.java | 2 +- .../server/resourcemanager/RMServerUtils.java | 27 +- .../capacity/IntraQueueCandidatesSelector.java | 5 + .../server/resourcemanager/rmapp/RMAppImpl.java | 17 +- .../rmapp/attempt/RMAppAttemptImpl.java | 46 +- .../rmcontainer/RMContainer.java | 4 - .../RMContainerChangeResourceEvent.java | 44 -- .../rmcontainer/RMContainerImpl.java | 46 -- .../scheduler/AbstractYarnScheduler.java | 171 +++++-- .../scheduler/AppSchedulingInfo.java | 283 +---------- .../scheduler/ContainerUpdateContext.java | 193 ++++++-- .../resourcemanager/scheduler/QueueMetrics.java | 13 + .../scheduler/SchedulerApplicationAttempt.java | 217 +++----- .../scheduler/SchedulerNode.java | 44 -- .../scheduler/capacity/AbstractCSQueue.java | 13 +- .../scheduler/capacity/CSQueue.java | 15 - .../scheduler/capacity/CapacityScheduler.java | 190 ++----- .../scheduler/capacity/LeafQueue.java | 152 +----- .../scheduler/capacity/ParentQueue.java | 53 +- .../scheduler/capacity/UsersManager.java | 4 +- .../capacity/allocator/ContainerAllocator.java | 31 +- .../allocator/IncreaseContainerAllocator.java | 337 ------------- .../common/ContainerAllocationProposal.java | 9 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 245 +++------ .../common/fica/FiCaSchedulerNode.java | 14 - .../scheduler/fair/FSAppAttempt.java | 54 +- .../scheduler/fair/FSLeafQueue.java | 24 +- .../scheduler/fair/FSParentQueue.java | 21 + .../resourcemanager/scheduler/fair/FSQueue.java | 41 +- .../scheduler/fair/FairScheduler.java | 42 +- .../scheduler/fifo/FifoScheduler.java | 8 - .../security/NMTokenSecretManagerInRM.java | 4 +- .../yarn/server/resourcemanager/MockRM.java | 6 +- .../resourcemanager/TestClientRMService.java | 7 +- .../applicationsmanager/TestAMRestart.java | 84 +++- ...ionalCapacityPreemptionPolicyIntraQueue.java | 55 +++ .../resourcemanager/resource/TestResources.java | 4 +- .../TestRMAppAttemptImplDiagnostics.java | 2 +- .../attempt/TestRMAppAttemptTransitions.java | 19 +- .../capacity/TestCapacityScheduler.java | 200 ++++++++ .../scheduler/capacity/TestChildQueueOrder.java | 4 +- .../capacity/TestContainerResizing.java | 134 +---- .../capacity/TestIncreaseAllocationExpirer.java | 12 +- .../scheduler/capacity/TestLeafQueue.java | 4 +- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/fair/FairSchedulerTestBase.java | 2 + .../scheduler/fair/TestFairScheduler.java | 103 +++- .../fair/TestFairSchedulerPreemption.java | 12 +- .../reader/TimelineReaderServer.java | 9 + .../src/site/markdown/TimelineServiceV2.md | 9 + .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 50 +- .../src/main/webapp/app/adapters/yarn-app.js | 4 +- .../app/components/app-usage-donut-chart.js | 1 - .../main/webapp/app/components/donut-chart.js | 1 - .../webapp/app/controllers/app-table-columns.js | 98 ++++ .../webapp/app/controllers/yarn-apps/apps.js | 25 + .../app/controllers/yarn-apps/services.js | 25 + .../main/webapp/app/controllers/yarn-node.js | 3 +- .../webapp/app/controllers/yarn-nodes/table.js | 109 ++++ .../webapp/app/controllers/yarn-queue/apps.js | 31 ++ .../main/webapp/app/helpers/log-files-comma.js | 10 +- .../src/main/webapp/app/models/yarn-rm-node.js | 8 + .../src/main/webapp/app/routes/yarn-apps.js | 2 - .../main/webapp/app/routes/yarn-apps/apps.js | 9 + .../webapp/app/routes/yarn-apps/services.js | 11 + .../src/main/webapp/app/routes/yarn-queue.js | 4 +- .../src/main/webapp/app/routes/yarn-queues.js | 3 +- .../src/main/webapp/app/serializers/yarn-app.js | 3 +- .../app/serializers/yarn-node-container.js | 2 +- .../main/webapp/app/serializers/yarn-node.js | 4 +- .../main/webapp/app/serializers/yarn-rm-node.js | 4 +- .../webapp/app/templates/yarn-apps/apps.hbs | 13 +- .../webapp/app/templates/yarn-apps/services.hbs | 4 +- .../src/main/webapp/app/templates/yarn-node.hbs | 23 +- .../main/webapp/app/templates/yarn-nodes.hbs | 2 +- .../webapp/app/templates/yarn-nodes/table.hbs | 53 +- .../webapp/app/templates/yarn-queue/apps.hbs | 3 +- .../hadoop-yarn-ui/src/main/webapp/bower.json | 2 + .../hadoop-yarn-ui/src/main/webapp/package.json | 12 +- .../unit/controllers/app-table-columns-test.js | 30 ++ .../unit/controllers/yarn-apps/apps-test.js | 30 ++ .../unit/controllers/yarn-apps/services-test.js | 30 ++ .../unit/controllers/yarn-nodes/table-test.js | 30 ++ .../unit/controllers/yarn-queue/apps-test.js | 30 ++ pom.xml | 2 +- start-build-env.sh | 4 +- 308 files changed, 8137 insertions(+), 3303 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/566c8b74/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/566c8b74/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/566c8b74/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/566c8b74/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
