Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5dfae69 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5dfae69 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5dfae69 Branch: refs/heads/HDFS-7240 Commit: a5dfae69f5843cd613ad82a8a31545ee14f652ef Parents: faa01f3 9714fc1 Author: Anu Engineer <[email protected]> Authored: Tue Mar 13 18:16:12 2018 -0700 Committer: Anu Engineer <[email protected]> Committed: Tue Mar 13 18:16:12 2018 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/pom.xml | 26 +- .../src/main/conf/log4j.properties | 3 + .../org/apache/hadoop/http/HttpServer2.java | 12 +- .../AuthenticationFilterInitializer.java | 9 +- .../AuthenticationWithProxyUserFilter.java | 124 ----- .../src/main/resources/core-default.xml | 11 + .../org/apache/hadoop/http/TestHttpServer.java | 23 +- .../hadoop/http/TestHttpServerWithSpengo.java | 494 ------------------- .../org/apache/hadoop/log/TestLogLevel.java | 4 +- .../security/TestAuthenticationFilter.java | 13 +- .../TestAuthenticationWithProxyUserFilter.java | 79 --- .../apache/hadoop/test/GenericTestUtils.java | 68 ++- .../hadoop/crypto/key/kms/server/TestKMS.java | 6 +- .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../hadoop/hdfs/protocol/DatanodeInfo.java | 31 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 12 +- .../protocol/SnapshottableDirectoryStatus.java | 8 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 34 ++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 14 + .../hadoop/hdfs/web/resources/GetOpParam.java | 3 +- .../src/main/proto/hdfs.proto | 1 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +- .../federation/resolver/MountTableResolver.java | 18 +- .../federation/router/ConnectionManager.java | 59 ++- .../federation/router/ConnectionPool.java | 2 +- .../federation/router/ConnectionPoolId.java | 6 + .../federation/router/FederationUtil.java | 20 + .../federation/router/RouterQuotaManager.java | 14 +- .../federation/store/records/BaseRecord.java | 16 +- .../store/records/MembershipState.java | 29 +- .../federation/store/records/MountTable.java | 42 +- .../federation/store/records/RouterState.java | 9 +- .../hdfs/server/namenode/FSDirRenameOp.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 1 + .../hdfs/server/namenode/INodeDirectory.java | 10 +- .../hdfs/server/namenode/INodeReference.java | 4 - .../snapshot/DirectorySnapshottableFeature.java | 5 +- .../snapshot/DirectoryWithSnapshotFeature.java | 131 ++--- .../snapshot/FSImageFormatPBSnapshot.java | 6 +- .../namenode/snapshot/SnapshotDiffInfo.java | 11 +- .../snapshot/SnapshotDiffListingInfo.java | 15 +- .../snapshot/SnapshotFSImageFormat.java | 4 +- .../web/resources/NamenodeWebHdfsMethods.java | 7 + .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 12 +- .../java/org/apache/hadoop/hdfs/util/Diff.java | 131 +++-- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 23 + .../src/main/resources/hdfs-default.xml | 6 +- .../src/main/webapps/hdfs/dfshealth.html | 3 + .../src/main/webapps/hdfs/explorer.html | 3 + .../src/site/markdown/HDFSDiskbalancer.md | 6 +- .../src/site/markdown/HDFSRouterFederation.md | 12 +- .../hadoop-hdfs/src/site/markdown/ViewFs.md | 139 ++++++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 92 ++++ .../apache/hadoop/hdfs/StripedFileTestUtil.java | 15 + .../hdfs/TestFileStatusWithDefaultECPolicy.java | 107 ++++ .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 102 ---- .../hdfs/TestFileStatusWithRandomECPolicy.java | 49 ++ .../hadoop/hdfs/TestReconstructStripedFile.java | 48 +- ...econstructStripedFileWithRandomECPolicy.java | 49 ++ .../TestBalancerWithMultipleNameNodes.java | 10 +- .../blockmanagement/TestBlockInfoStriped.java | 33 +- .../blockmanagement/TestBlockStatsMXBean.java | 5 + .../TestLowRedundancyBlockQueues.java | 19 +- .../TestPendingReconstruction.java | 10 +- .../datanode/TestDataNodeVolumeFailure.java | 37 +- .../resolver/TestMountTableResolver.java | 69 +++ .../router/TestConnectionManager.java | 157 ++++++ .../federation/router/TestRouterAdminCLI.java | 38 +- .../federation/router/TestRouterQuota.java | 2 +- .../router/TestRouterQuotaManager.java | 12 + .../federation/router/TestRouterSafemode.java | 9 + .../store/records/TestMountTable.java | 43 ++ .../hdfs/server/namenode/TestFSImage.java | 9 + .../namenode/snapshot/SnapshotTestHelper.java | 79 +-- .../snapshot/TestRenameWithSnapshots.java | 129 ++--- .../snapshot/TestSetQuotaWithSnapshot.java | 6 +- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 38 ++ .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 186 ++++++- .../test/resources/testErasureCodingConf.xml | 28 +- .../src/test/resources/testHDFSConf.xml | 4 + .../mapreduce/v2/app/webapp/AppController.java | 7 +- .../src/main/resources/mapred-default.xml | 16 - .../plugin/paralleltests/CreateDirsMojo.java | 100 ++++ hadoop-project/src/site/markdown/index.md.vm | 2 +- hadoop-tools/hadoop-aws/pom.xml | 26 +- .../org/apache/hadoop/fs/s3a/Constants.java | 11 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 38 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 1 + .../org/apache/hadoop/fs/s3a/Statistic.java | 2 + .../src/site/markdown/tools/hadoop-aws/index.md | 41 +- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 53 +- .../org/apache/hadoop/tools/DistCpOptions.java | 5 - .../tools/mapred/RetriableFileCopyCommand.java | 53 +- .../apache/hadoop/tools/util/DistCpUtils.java | 2 +- .../hadoop/tools/util/ThrottledInputStream.java | 48 +- .../hadoop/tools/mapred/TestCopyMapper.java | 38 +- .../hadoop/yarn/conf/YarnConfiguration.java | 27 + .../hadoop/yarn/service/webapp/ApiServer.java | 4 +- .../definition/YARN-Services-Examples.md | 10 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 7 + .../hadoop/yarn/service/TestApiServer.java | 1 + .../src/test/resources/example-app.json | 1 + .../examples/httpd-no-dns/httpd-no-dns.json | 1 + .../examples/httpd/httpd.json | 1 + .../examples/sleeper/sleeper.json | 1 + .../yarn/service/api/records/Container.java | 12 +- .../yarn/service/api/records/Service.java | 46 +- .../exceptions/RestApiErrorMessages.java | 2 + .../yarn/service/utils/ServiceApiUtil.java | 6 + .../hadoop/yarn/service/ServiceTestUtils.java | 1 + .../hadoop/yarn/service/TestServiceApiUtil.java | 13 + .../yarn/service/TestYarnNativeServices.java | 7 + .../src/test/resources/example-app.json | 1 + .../service/conf/examples/app-override.json | 1 + .../hadoop/yarn/service/conf/examples/app.json | 1 + .../yarn/service/conf/examples/default.json | 1 + .../yarn/service/conf/examples/external0.json | 1 + .../yarn/service/conf/examples/external1.json | 1 + .../yarn/service/conf/examples/external2.json | 1 + .../src/main/resources/yarn-default.xml | 51 ++ .../RegisterNodeManagerRequest.java | 5 - .../pb/RegisterNodeManagerRequestPBImpl.java | 79 --- .../hadoop/yarn/server/webapp/AppBlock.java | 113 +++-- .../yarn_server_common_service_protos.proto | 1 - .../hadoop/yarn/server/nodemanager/Context.java | 4 +- .../nodemanager/LinuxContainerExecutor.java | 18 +- .../yarn/server/nodemanager/NodeManager.java | 12 - .../nodemanager/NodeStatusUpdaterImpl.java | 14 - .../containermanager/ContainerManagerImpl.java | 15 - .../linux/privileged/PrivilegedOperation.java | 3 +- .../linux/resources/ResourceHandlerModule.java | 10 + .../linux/resources/numa/NumaNodeResource.java | 204 ++++++++ .../resources/numa/NumaResourceAllocation.java | 69 +++ .../resources/numa/NumaResourceAllocator.java | 342 +++++++++++++ .../resources/numa/NumaResourceHandlerImpl.java | 108 ++++ .../linux/resources/numa/package-info.java | 28 ++ .../logaggregation/AppLogAggregatorImpl.java | 11 +- .../tracker/NMLogAggregationStatusTracker.java | 244 --------- .../amrmproxy/BaseAMRMProxyTest.java | 6 - .../numa/TestNumaResourceAllocator.java | 281 +++++++++++ .../numa/TestNumaResourceHandlerImpl.java | 181 +++++++ .../TestNMLogAggregationStatusTracker.java | 124 ----- .../resourcemanager/ResourceTrackerService.java | 17 +- .../resourcemanager/rmnode/RMNodeImpl.java | 6 - .../rmnode/RMNodeStartedEvent.java | 11 - .../scheduler/fair/MaxRunningAppsEnforcer.java | 3 +- ...pportunisticContainerAllocatorAMService.java | 29 +- ...neReaderAuthenticationFilterInitializer.java | 10 +- .../markdown/yarn-service/YarnServiceAPI.md | 12 +- .../main/webapp/app/models/yarn-app-timeline.js | 1 - .../src/main/webapp/app/models/yarn-app.js | 5 - .../webapp/app/serializers/yarn-app-timeline.js | 1 - .../src/main/webapp/app/serializers/yarn-app.js | 1 - .../src/main/webapp/app/templates/yarn-app.hbs | 2 - 155 files changed, 3690 insertions(+), 2060 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5dfae69/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
