Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0becabce Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0becabce Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0becabce Branch: refs/heads/HDFS-7240 Commit: 0becabcefde6abda467819f879c76e710f99adca Parents: 4c03c58 46d29e3 Author: Anu Engineer <[email protected]> Authored: Wed Mar 7 12:53:04 2018 -0800 Committer: Anu Engineer <[email protected]> Committed: Wed Mar 7 12:53:04 2018 -0800 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 2 +- .../main/resources/checkstyle/checkstyle.xml | 5 +- .../java/org/apache/hadoop/fs/FileStatus.java | 2 +- .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../crypto/key/kms/server/TestKMSWithZK.java | 39 - .../org/apache/hadoop/minikdc/TestMiniKdc.java | 3 +- .../hadoop/hdfs/DistributedFileSystem.java | 65 ++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 46 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 28 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 1 + .../hadoop/fs/http/server/FSOperations.java | 20 +- .../http/server/HttpFSParametersProvider.java | 30 +- .../hadoop/fs/http/server/HttpFSServer.java | 18 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 174 +++++ .../http/server/TestHttpFSServerWebServer.java | 10 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 + .../NamenodeProtocolServerSideTranslatorPB.java | 3 +- .../NamenodeProtocolTranslatorPB.java | 5 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 2 +- .../hdfs/server/balancer/NameNodeConnector.java | 5 +- .../server/blockmanagement/BlockManager.java | 17 +- .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../hdfs/server/datanode/DiskBalancer.java | 4 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 3 +- .../diskbalancer/command/ExecuteCommand.java | 20 +- .../hdfs/server/federation/router/Router.java | 18 +- .../hdfs/server/namenode/FSNamesystem.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 35 +- .../snapshot/AbstractINodeDiffList.java | 26 +- .../namenode/snapshot/DiffListBySkipList.java | 585 +++++++++++++++ .../namenode/snapshot/DirectoryDiffList.java | 545 -------------- .../snapshot/DirectoryDiffListFactory.java | 70 ++ .../snapshot/DirectoryWithSnapshotFeature.java | 42 +- .../namenode/snapshot/SnapshotManager.java | 15 +- .../web/resources/NamenodeWebHdfsMethods.java | 5 +- .../hdfs/server/protocol/NamenodeProtocol.java | 9 +- .../hadoop/hdfs/tools/DiskBalancerCLI.java | 15 + .../src/main/proto/NamenodeProtocol.proto | 1 + .../src/main/resources/hdfs-default.xml | 19 +- .../src/site/markdown/HDFSRouterFederation.md | 5 + ...estDFSInotifyEventInputStreamKerberized.java | 223 ++++++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 44 ++ .../org/apache/hadoop/hdfs/TestGetBlocks.java | 50 +- .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 21 +- .../hdfs/server/balancer/TestBalancer.java | 2 +- .../diskbalancer/DiskBalancerTestUtil.java | 21 +- .../server/diskbalancer/TestDiskBalancer.java | 72 +- .../diskbalancer/TestDiskBalancerRPC.java | 4 +- .../command/TestDiskBalancerCommand.java | 40 + .../router/TestRouterNamenodeMonitoring.java | 143 ++++ .../snapshot/TestDiffListBySkipList.java | 357 +++++++++ .../snapshot/TestDirectoryDiffList.java | 264 ------- .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 57 ++ .../org/apache/hadoop/examples/pi/package.html | 2 +- hadoop-project/pom.xml | 72 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 2 + .../hadoop-aws/dev-support/findbugs-exclude.xml | 36 + .../hadoop/fs/s3a/FailureInjectionPolicy.java | 163 +++++ .../fs/s3a/InconsistentAmazonS3Client.java | 177 ++--- .../hadoop/fs/s3a/InconsistentS3Object.java | 232 ++++++ .../java/org/apache/hadoop/fs/s3a/Invoker.java | 5 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 44 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 122 ++-- .../org/apache/hadoop/fs/s3a/S3AOpContext.java | 87 +++ .../apache/hadoop/fs/s3a/S3AReadOpContext.java | 56 ++ .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 45 +- .../hadoop/fs/s3a/S3GuardExistsRetryPolicy.java | 47 ++ .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 20 +- .../hadoop/fs/s3a/s3guard/MetadataStore.java | 3 + .../site/markdown/tools/hadoop-aws/testing.md | 5 + .../hadoop/fs/s3a/ITestS3AInconsistency.java | 116 ++- .../fs/s3a/ITestS3GuardListConsistency.java | 13 +- .../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 6 + .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 4 +- .../fs/s3a/commit/AbstractCommitITest.java | 3 +- .../ITestDynamoDBMetadataStoreScale.java | 174 +++++ .../fs/s3a/s3guard/MetadataStoreTestBase.java | 2 +- .../scale/ITestDynamoDBMetadataStoreScale.java | 48 -- .../scale/ITestS3AHugeFilesSSECDiskBlocks.java | 58 ++ .../swift/TestSwiftFileSystemBlockLocation.java | 2 +- .../api/records/AllocationTagNamespace.java | 336 +++++++++ .../api/records/AllocationTagNamespaceType.java | 74 ++ .../hadoop/yarn/api/records/AllocationTags.java | 50 ++ .../hadoop/yarn/api/records/Evaluable.java | 38 + .../yarn/api/records/TargetApplications.java | 53 ++ .../yarn/api/resource/PlacementConstraints.java | 26 +- .../InvalidAllocationTagException.java | 34 + .../constraint/PlacementConstraintParser.java | 4 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 10 + .../LogAggregationIndexedFileController.java | 60 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 22 +- .../yarn/util/AuxiliaryServiceHelper.java | 2 +- .../TestLogAggregationIndexFileController.java | 54 ++ .../application_123456_0001.har/_SUCCESS | 0 .../application_123456_0001.har/_index | 3 + .../application_123456_0001.har/_masterindex | 2 + .../application_123456_0001.har/part-0 | Bin 0 -> 4123 bytes .../RegisterNodeManagerRequest.java | 5 + .../pb/RegisterNodeManagerRequestPBImpl.java | 79 ++ .../yarn_server_common_service_protos.proto | 1 + .../server/nodemanager/ContainerExecutor.java | 64 +- .../hadoop/yarn/server/nodemanager/Context.java | 4 +- .../nodemanager/LinuxContainerExecutor.java | 8 - .../yarn/server/nodemanager/NodeManager.java | 12 + .../nodemanager/NodeStatusUpdaterImpl.java | 14 + .../containermanager/ContainerManagerImpl.java | 15 + .../launcher/ContainerLaunch.java | 87 ++- .../runtime/DefaultLinuxContainerRuntime.java | 6 - .../DelegatingLinuxContainerRuntime.java | 11 - .../runtime/DockerLinuxContainerRuntime.java | 7 - .../logaggregation/AppLogAggregatorImpl.java | 11 +- .../tracker/NMLogAggregationStatusTracker.java | 244 +++++++ .../runtime/ContainerRuntime.java | 11 - .../container-executor/impl/utils/docker-util.c | 72 +- .../test/utils/test_docker_util.cc | 53 +- .../amrmproxy/BaseAMRMProxyTest.java | 6 + .../launcher/TestContainerLaunch.java | 135 +++- .../TestNMLogAggregationStatusTracker.java | 124 ++++ .../resourcemanager/ResourceTrackerService.java | 17 +- .../resourcemanager/rmnode/RMNodeImpl.java | 6 + .../rmnode/RMNodeStartedEvent.java | 11 + .../constraint/AllocationTagsManager.java | 9 + .../InvalidAllocationTagsQueryException.java | 4 + .../constraint/PlacementConstraintsUtil.java | 71 +- .../SingleConstraintAppPlacementAllocator.java | 31 +- .../constraint/TestAllocationTagsNamespace.java | 147 ++++ .../TestPlacementConstraintsUtil.java | 151 ++++ .../pom.xml | 78 +- .../storage/flow/TestHBaseStorageFlowRun.java | 19 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 103 +-- .../pom.xml | 6 +- .../pom.xml | 165 +++++ .../src/assembly/coprocessor.xml | 38 + .../common/HBaseTimelineServerUtils.java | 222 ++++++ .../storage/common/package-info.java | 28 + .../storage/flow/FlowRunCoprocessor.java | 278 +++++++ .../storage/flow/FlowScanner.java | 723 +++++++++++++++++++ .../storage/flow/FlowScannerOperation.java | 46 ++ .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../pom.xml | 172 +++++ .../src/assembly/coprocessor.xml | 38 + .../common/HBaseTimelineServerUtils.java | 224 ++++++ .../storage/common/package-info.java | 28 + .../storage/flow/FlowRunCoprocessor.java | 285 ++++++++ .../storage/flow/FlowScanner.java | 723 +++++++++++++++++++ .../storage/flow/FlowScannerOperation.java | 46 ++ .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../pom.xml | 184 ++--- .../src/assembly/coprocessor.xml | 37 - .../common/HBaseTimelineServerUtils.java | 135 ---- .../storage/common/package-info.java | 28 - .../storage/flow/FlowRunCoprocessor.java | 277 ------- .../storage/flow/FlowScanner.java | 723 ------------------- .../storage/flow/FlowScannerOperation.java | 46 -- .../storage/flow/package-info.java | 29 - .../timelineservice/storage/package-info.java | 28 - .../src/site/markdown/Federation.md | 2 + pom.xml | 4 +- 160 files changed, 8800 insertions(+), 2967 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0becabce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0becabce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0becabce/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
