Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df3ff904 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df3ff904 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df3ff904 Branch: refs/heads/HDFS-7240 Commit: df3ff9042a6327b784ecf90ea8be8f0fe567859e Parents: bb3c07f 8ab776d Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Apr 10 12:22:50 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Tue Apr 10 12:22:50 2018 -0700 ---------------------------------------------------------------------- BUILDING.txt | 14 + .../src/main/bin/hadoop-functions.sh | 9 +- .../apache/hadoop/crypto/key/KeyProvider.java | 11 +- .../fs/CommonConfigurationKeysPublic.java | 21 + .../main/java/org/apache/hadoop/ipc/RPC.java | 35 +- .../org/apache/hadoop/net/NetworkTopology.java | 106 +- .../hadoop/util/concurrent/HadoopExecutors.java | 9 +- .../src/site/markdown/HttpAuthentication.md | 2 +- .../markdown/release/3.1.0/CHANGES.3.1.0.md | 1022 + .../release/3.1.0/RELEASENOTES.3.1.0.md | 199 + .../fs/contract/AbstractContractCreateTest.java | 12 +- .../java/org/apache/hadoop/io/TestIOUtils.java | 2 +- .../hadoop/hdfs/protocol/AclException.java | 10 + .../ha/RequestHedgingProxyProvider.java | 3 + .../ha/TestRequestHedgingProxyProvider.java | 34 + .../federation/metrics/NamenodeBeanMetrics.java | 3 + .../federation/router/ConnectionContext.java | 35 +- .../federation/router/ConnectionManager.java | 10 +- .../federation/router/ConnectionPool.java | 98 +- .../federation/router/ConnectionPoolId.java | 19 +- .../server/federation/router/RemoteMethod.java | 68 +- .../router/RouterNamenodeProtocol.java | 187 + .../federation/router/RouterRpcClient.java | 62 +- .../federation/router/RouterRpcServer.java | 141 +- .../router/SubClusterTimeoutException.java | 33 + .../driver/impl/StateStoreFileSystemImpl.java | 6 +- .../server/federation/MiniRouterDFSCluster.java | 39 +- .../router/TestConnectionManager.java | 56 +- .../server/federation/router/TestRouter.java | 70 +- .../federation/router/TestRouterQuota.java | 4 + .../router/TestRouterRPCClientRetries.java | 126 +- .../server/federation/router/TestRouterRpc.java | 136 +- .../src/test/resources/contract/webhdfs.xml | 5 + .../jdiff/Apache_Hadoop_HDFS_3.1.0.xml | 676 + .../server/blockmanagement/BlockIdManager.java | 17 + .../server/blockmanagement/BlockManager.java | 5 +- .../blockmanagement/BlockManagerSafeMode.java | 2 +- .../hdfs/server/blockmanagement/BlocksMap.java | 12 +- .../blockmanagement/CorruptReplicasMap.java | 35 +- .../blockmanagement/InvalidateBlocks.java | 13 +- .../server/namenode/EncryptionZoneManager.java | 8 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 12 + .../hdfs/server/namenode/FSTreeTraverser.java | 339 + .../server/namenode/ReencryptionHandler.java | 615 +- .../server/namenode/ReencryptionUpdater.java | 2 +- .../src/site/markdown/ArchivalStorage.md | 2 +- .../src/site/markdown/MemoryStorage.md | 2 +- .../blockmanagement/TestBlockManager.java | 61 +- .../blockmanagement/TestCorruptReplicaInfo.java | 48 +- .../hdfs/server/namenode/TestReencryption.java | 3 - .../namenode/TestReencryptionHandler.java | 10 +- .../apache/hadoop/net/TestNetworkTopology.java | 75 +- .../src/test/resources/testCryptoConf.xml | 19 + .../Apache_Hadoop_MapReduce_Common_3.1.0.xml | 113 + .../Apache_Hadoop_MapReduce_Core_3.1.0.xml | 28075 +++++++++++++++++ .../Apache_Hadoop_MapReduce_JobClient_3.1.0.xml | 16 + .../jobhistory/JobHistoryEventHandler.java | 2 +- hadoop-project/src/site/site.xml | 4 + .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 18 +- .../fs/s3a/s3guard/LocalMetadataStore.java | 17 +- .../hadoop/fs/s3a/s3guard/MetadataStore.java | 12 + .../fs/s3a/s3guard/NullMetadataStore.java | 4 + .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 14 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 11 +- .../s3guard/AbstractS3GuardToolTestBase.java | 21 +- .../dev-support/findbugs-exclude.xml | 7 + .../jdiff/Apache_Hadoop_YARN_Client_3.1.0.xml | 3146 ++ .../jdiff/Apache_Hadoop_YARN_Common_3.1.0.xml | 3034 ++ .../Apache_Hadoop_YARN_Server_Common_3.1.0.xml | 1331 + .../api/records/AllocationTagNamespaceType.java | 2 +- .../timelineservice/SubApplicationEntity.java | 50 + .../hadoop/yarn/conf/YarnConfiguration.java | 42 + .../hadoop-yarn-services-api/pom.xml | 5 + .../client/SystemServiceManagerImpl.java | 381 + .../service/client/TestSystemServiceImpl.java | 180 + .../users/sync/user1/example-app1.yarnfile | 16 + .../users/sync/user1/example-app2.yarnfile | 16 + .../users/sync/user1/example-app3.json | 16 + .../users/sync/user2/example-app1.yarnfile | 16 + .../users/sync/user2/example-app2.yarnfile | 16 + .../yarn/service/conf/YarnServiceConf.java | 2 + .../yarn/service/TestSystemServiceManager.java | 156 + .../yarn/client/api/TimelineV2Client.java | 47 +- .../client/api/impl/TimelineClientImpl.java | 23 +- .../client/api/impl/TimelineV2ClientImpl.java | 30 +- .../LogAggregationIndexedFileController.java | 3 +- .../yarn/util/timeline/TimelineUtils.java | 3 +- .../src/main/resources/yarn-default.xml | 8 + .../server/service/SystemServiceManager.java | 25 + .../yarn/server/service/package-info.java | 27 + .../nodemanager/amrmproxy/AMRMProxyService.java | 2 +- .../containermanager/AuxServices.java | 160 +- .../containermanager/ContainerManagerImpl.java | 3 +- .../runtime/DockerLinuxContainerRuntime.java | 8 +- .../localizer/ResourceLocalizationService.java | 20 + .../monitor/ContainerMetrics.java | 2 +- .../amrmproxy/BaseAMRMProxyTest.java | 5 + .../amrmproxy/TestAMRMProxyService.java | 42 + .../containermanager/TestAuxServices.java | 167 +- .../runtime/TestDockerContainerRuntime.java | 14 +- .../TestResourceLocalizationService.java | 125 + .../monitor/TestContainerMetrics.java | 17 + .../server/resourcemanager/RMServerUtils.java | 5 +- .../server/resourcemanager/ResourceManager.java | 30 +- .../rmapp/attempt/RMAppAttemptImpl.java | 47 + .../resourcemanager/scheduler/Allocation.java | 12 + .../scheduler/SchedulerUtils.java | 33 +- .../capacity/AutoCreatedLeafQueue.java | 3 +- .../AutoCreatedQueueManagementPolicy.java | 12 +- .../scheduler/capacity/CapacityScheduler.java | 2 + .../CapacitySchedulerConfiguration.java | 28 + .../scheduler/capacity/LeafQueue.java | 11 + .../scheduler/capacity/ManagedParentQueue.java | 5 +- .../GuaranteedOrZeroCapacityOverTimePolicy.java | 573 +- .../constraint/AllocationTagNamespace.java | 312 - .../scheduler/constraint/AllocationTags.java | 44 +- .../constraint/AllocationTagsManager.java | 47 +- .../constraint/PlacementConstraintsUtil.java | 41 +- .../constraint/TargetApplications.java | 53 +- .../constraint/TargetApplicationsNamespace.java | 326 + .../placement/PendingAskUpdateResult.java | 8 + .../SingleConstraintAppPlacementAllocator.java | 21 - .../yarn/server/resourcemanager/MockNM.java | 15 + .../server/resourcemanager/TestAppManager.java | 20 +- .../resourcemanager/TestRMEmbeddedElector.java | 72 +- .../TestCombinedSystemMetricsPublisher.java | 26 + .../server/resourcemanager/rmapp/MockRMApp.java | 9 +- ...stCapacitySchedulerAutoCreatedQueueBase.java | 241 +- .../TestCapacitySchedulerAutoQueueCreation.java | 233 +- .../TestQueueManagementDynamicEditPolicy.java | 30 +- ...estSchedulingRequestContainerAllocation.java | 5 +- .../constraint/TestAllocationTagsManager.java | 22 +- .../constraint/TestAllocationTagsNamespace.java | 89 +- .../TestPlacementConstraintsUtil.java | 125 +- ...stTimelineReaderWebServicesHBaseStorage.java | 7 +- .../TestHBaseTimelineStorageEntities.java | 3 +- .../storage/HBaseTimelineWriterImpl.java | 3 +- .../collector/TimelineCollectorWebService.java | 19 +- .../reader/TimelineParserForCompareExpr.java | 7 +- .../reader/TimelineParserForEqualityExpr.java | 7 +- .../reader/TimelineReaderServer.java | 2 +- .../TestTimelineReaderWebServicesUtils.java | 25 + .../src/site/markdown/TimelineServiceV2.md | 8 +- .../webapp/app/components/node-menu-panel.js | 25 + .../main/webapp/app/components/timeline-view.js | 35 +- .../webapp/app/controllers/yarn-app-attempt.js | 9 +- .../webapp/app/controllers/yarn-app/attempts.js | 11 +- .../webapp/app/controllers/yarn-node-app.js | 3 +- .../webapp/app/controllers/yarn-node-apps.js | 3 +- .../app/controllers/yarn-node-container.js | 3 +- .../app/controllers/yarn-node-containers.js | 3 +- .../main/webapp/app/controllers/yarn-node.js | 3 +- .../webapp/app/controllers/yarn-nodes/table.js | 2 +- .../app/controllers/yarn-tools/yarn-conf.js | 10 +- .../src/main/webapp/app/helpers/node-link.js | 2 +- .../src/main/webapp/app/initializers/loader.js | 3 + .../main/webapp/app/routes/yarn-node-apps.js | 8 +- .../webapp/app/routes/yarn-node-containers.js | 8 +- .../src/main/webapp/app/routes/yarn-node.js | 8 +- .../templates/components/node-menu-panel.hbs | 8 +- .../app/templates/components/timeline-view.hbs | 12 +- .../webapp/app/templates/yarn-app-attempt.hbs | 2 + .../webapp/app/templates/yarn-app/attempts.hbs | 2 + .../app/templates/yarn-tools/yarn-conf.hbs | 6 +- 164 files changed, 43124 insertions(+), 1620 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df3ff904/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df3ff904/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java index 3c31a8c,9cfadc7..5a5082a --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java @@@ -810,14 -810,34 +810,33 @@@ public class RPC /** An RPC Server. */ public abstract static class Server extends org.apache.hadoop.ipc.Server { - boolean verbose; - + boolean verbose; + private static final Pattern COMPLEX_SERVER_NAME_PATTERN = + Pattern.compile("(?:[^\\$]*\\$)*([A-Za-z][^\\$]+)(?:\\$\\d+)?"); + + /** + * Get a meaningful and short name for a server based on a java class. + * + * The rules are defined to support the current naming schema of the + * generated protobuf classes where the final class usually an anonymous + * inner class of an inner class. + * + * 1. For simple classes it returns with the simple name of the classes + * (with the name without package name) + * + * 2. For inner classes, this is the simple name of the inner class. + * + * 3. If it is an Object created from a class factory + * E.g., org.apache.hadoop.ipc.TestRPC$TestClass$2 + * this method returns parent class TestClass. + * + * 4. If it is an anonymous class E.g., 'org.apache.hadoop.ipc.TestRPC$10' + * serverNameFromClass returns parent class TestRPC. + * + * + */ static String serverNameFromClass(Class<?> clazz) { - //The basic idea here is to handle names like - //org.apache.hadoop.hdsl.protocol.proto. - // - // StorageDatanodeProtocolProtos$StorageContainerDatanodeProtocolService$2 - //where the getSimpleName is also empty String name = clazz.getName(); String[] names = clazz.getName().split("\\.", -1); if (names != null && names.length > 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/df3ff904/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df3ff904/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- diff --cc hadoop-project/src/site/site.xml index b3d2df0,fdf5583..9f96584 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@@ -106,17 -106,9 +106,18 @@@ <item name="Upgrade Domain" href="hadoop-project-dist/hadoop-hdfs/HdfsUpgradeDomain.html"/> <item name="DataNode Admin" href="hadoop-project-dist/hadoop-hdfs/HdfsDataNodeAdminGuide.html"/> <item name="Router Federation" href="hadoop-project-dist/hadoop-hdfs-rbf/HDFSRouterFederation.html"/> + <item name="Provided Storage" href="hadoop-project-dist/hadoop-hdfs/HdfsProvidedStorage.html"/> </menu> + <menu name="Ozone" inherit="top"> + <item name="Getting Started" href="hadoop-project-dist/hadoop-hdfs/OzoneGettingStarted.html"/> + <item name="Ozone Overview" + href="hadoop-project-dist/hadoop-hdfs/OzoneOverview.html"/> + <item name="Commands Reference" href="hadoop-project-dist/hadoop-hdfs/OzoneCommandShell.html"/> + <item name="Ozone Rest API" href="hadoop-project-dist/hadoop-hdfs/OzoneRest.html"/> + <item name="Ozone Metrics" href="hadoop-project-dist/hadoop-hdfs/OzoneMetrics.html"/> + </menu> + <menu name="MapReduce" inherit="top"> <item name="Tutorial" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html"/> <item name="Commands Reference" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html"/> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
