Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efb79f4e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efb79f4e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efb79f4e Branch: refs/heads/HDFS-12943 Commit: efb79f4ead5ba67ac70ff87cc75f6a5a303598dc Parents: 3a5b78a 8f97d6f Author: Chao Sun <[email protected]> Authored: Tue Oct 23 22:33:29 2018 -0700 Committer: Chao Sun <[email protected]> Committed: Tue Oct 23 22:33:29 2018 -0700 ---------------------------------------------------------------------- LICENSE.txt | 1 + NOTICE.txt | 8 + dev-support/bin/dist-layout-stitching | 1 + dev-support/docker/Dockerfile | 1 + .../assemblies/hadoop-registry-dist.xml | 41 + .../hadoop-client-minicluster/pom.xml | 2 +- .../security/authentication/util/AuthToken.java | 2 +- .../hadoop-common/src/main/bin/hadoop | 6 + .../hadoop-common/src/main/conf/hadoop-env.sh | 13 + .../fs/CommonConfigurationKeysPublic.java | 19 +- .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 13 + .../io/compress/zstd/ZStandardCompressor.java | 2 +- .../io/compress/zstd/ZStandardDecompressor.java | 4 +- .../main/java/org/apache/hadoop/ipc/Server.java | 114 +- .../hadoop/security/SaslPropertiesResolver.java | 4 +- .../security/token/DelegationTokenIssuer.java | 111 ++ .../security/token/DelegationTokenIssuer.java | 111 -- .../io/compress/zstd/ZStandardCompressor.c | 21 +- .../io/compress/zstd/ZStandardDecompressor.c | 12 +- .../src/main/resources/core-default.xml | 21 +- .../site/markdown/registry/hadoop-registry.md | 1018 ++++++++++ .../src/site/markdown/registry/index.md | 31 + .../markdown/registry/registry-configuration.md | 397 ++++ .../src/site/markdown/registry/registry-dns.md | 224 +++ .../site/markdown/registry/registry-security.md | 120 ++ .../using-the-hadoop-service-registry.md | 273 +++ .../conf/TestCommonConfigurationFields.java | 3 +- .../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 15 + .../TestZStandardCompressorDecompressor.java | 10 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 53 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 105 + .../key/kms/server/KMSAuthenticationFilter.java | 23 +- .../kms/server/TestKMSAuthenticationFilter.java | 48 + hadoop-common-project/hadoop-registry/pom.xml | 299 +++ .../apache/hadoop/registry/cli/RegistryCli.java | 497 +++++ .../hadoop/registry/client/api/BindFlags.java | 41 + .../registry/client/api/DNSOperations.java | 60 + .../client/api/DNSOperationsFactory.java | 78 + .../registry/client/api/RegistryConstants.java | 388 ++++ .../registry/client/api/RegistryOperations.java | 182 ++ .../client/api/RegistryOperationsFactory.java | 160 ++ .../registry/client/api/package-info.java | 35 + .../registry/client/binding/JsonSerDeser.java | 117 ++ .../client/binding/RegistryPathUtils.java | 238 +++ .../client/binding/RegistryTypeUtils.java | 291 +++ .../registry/client/binding/RegistryUtils.java | 399 ++++ .../registry/client/binding/package-info.java | 22 + .../AuthenticationFailedException.java | 39 + .../exceptions/InvalidPathnameException.java | 40 + .../exceptions/InvalidRecordException.java | 41 + .../NoChildrenForEphemeralsException.java | 48 + .../exceptions/NoPathPermissionsException.java | 45 + .../client/exceptions/NoRecordException.java | 45 + .../client/exceptions/RegistryIOException.java | 58 + .../client/exceptions/package-info.java | 33 + .../impl/FSRegistryOperationsService.java | 248 +++ .../client/impl/RegistryOperationsClient.java | 55 + .../registry/client/impl/package-info.java | 26 + .../client/impl/zk/BindingInformation.java | 41 + .../registry/client/impl/zk/CuratorService.java | 896 +++++++++ .../registry/client/impl/zk/ListenerHandle.java | 25 + .../registry/client/impl/zk/PathListener.java | 30 + .../client/impl/zk/RegistryBindingSource.java | 36 + .../impl/zk/RegistryInternalConstants.java | 81 + .../impl/zk/RegistryOperationsService.java | 165 ++ .../client/impl/zk/RegistrySecurity.java | 1143 +++++++++++ .../registry/client/impl/zk/ZKPathDumper.java | 133 ++ .../client/impl/zk/ZookeeperConfigOptions.java | 118 ++ .../registry/client/impl/zk/package-info.java | 39 + .../registry/client/types/AddressTypes.java | 94 + .../hadoop/registry/client/types/Endpoint.java | 247 +++ .../registry/client/types/ProtocolTypes.java | 99 + .../client/types/RegistryPathStatus.java | 123 ++ .../registry/client/types/ServiceRecord.java | 312 +++ .../registry/client/types/package-info.java | 41 + .../client/types/yarn/PersistencePolicies.java | 50 + .../types/yarn/YarnRegistryAttributes.java | 42 + .../registry/conf/RegistryConfiguration.java | 41 + .../hadoop/registry/conf/package-info.java | 22 + .../dns/ApplicationServiceRecordProcessor.java | 366 ++++ .../server/dns/BaseServiceRecordProcessor.java | 470 +++++ .../dns/ContainerServiceRecordProcessor.java | 284 +++ .../hadoop/registry/server/dns/LookupTask.java | 39 + .../dns/PrivilegedRegistryDNSStarter.java | 81 + .../server/dns/RecordCreatorFactory.java | 275 +++ .../hadoop/registry/server/dns/RegistryDNS.java | 1800 ++++++++++++++++++ .../registry/server/dns/RegistryDNSServer.java | 267 +++ .../registry/server/dns/ReverseZoneUtils.java | 171 ++ .../registry/server/dns/SecureableZone.java | 151 ++ .../server/dns/ServiceRecordProcessor.java | 53 + .../registry/server/dns/ZoneSelector.java | 33 + .../registry/server/dns/package-info.java | 26 + .../integration/SelectByYarnPersistence.java | 60 + .../server/integration/package-info.java | 23 + .../hadoop/registry/server/package-info.java | 27 + .../server/services/AddingCompositeService.java | 56 + .../services/DeleteCompletionCallback.java | 57 + .../server/services/MicroZookeeperService.java | 282 +++ .../services/MicroZookeeperServiceKeys.java | 69 + .../server/services/RegistryAdminService.java | 529 +++++ .../registry/server/services/package-info.java | 39 + .../src/main/tla/hadoopregistry.tla | 582 ++++++ .../hadoop/registry/AbstractRegistryTest.java | 118 ++ .../hadoop/registry/AbstractZKRegistryTest.java | 113 ++ .../hadoop/registry/RegistryTestHelper.java | 353 ++++ .../hadoop/registry/cli/TestRegistryCli.java | 197 ++ .../client/binding/TestMarshalling.java | 143 ++ .../binding/TestRegistryOperationUtils.java | 56 + .../client/binding/TestRegistryPathUtils.java | 186 ++ .../client/impl/CuratorEventCatcher.java | 68 + .../client/impl/TestCuratorService.java | 249 +++ .../impl/TestFSRegistryOperationsService.java | 298 +++ .../client/impl/TestMicroZookeeperService.java | 61 + .../integration/TestYarnPolicySelector.java | 65 + .../operations/TestRegistryOperations.java | 331 ++++ .../secure/AbstractSecureRegistryTest.java | 369 ++++ .../registry/secure/KerberosConfiguration.java | 100 + .../secure/TestRegistrySecurityHelper.java | 211 ++ .../registry/secure/TestSecureLogins.java | 234 +++ .../registry/secure/TestSecureRegistry.java | 213 +++ .../registry/server/dns/TestRegistryDNS.java | 725 +++++++ .../server/dns/TestReverseZoneUtils.java | 89 + .../server/dns/TestSecureRegistryDNS.java | 44 + .../test/resources/0.17.172.in-addr.arpa.zone | 36 + .../src/test/resources/log4j.properties | 63 + .../src/test/resources/test.private | 32 + hadoop-common-project/pom.xml | 1 + .../hadoop/hdds/scm/XceiverClientGrpc.java | 194 +- .../hadoop/hdds/scm/XceiverClientManager.java | 14 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 25 + .../scm/client/ContainerOperationClient.java | 2 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 43 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 + .../hadoop/hdds/scm/client/ScmClient.java | 2 +- .../hdds/scm/container/ContainerException.java | 46 + .../hadoop/hdds/scm/container/ContainerID.java | 28 +- .../hdds/scm/container/ContainerInfo.java | 449 +++++ .../container/ContainerNotFoundException.java | 44 + .../ContainerReplicaNotFoundException.java | 45 + .../container/common/helpers/ContainerInfo.java | 482 ----- .../common/helpers/ContainerWithPipeline.java | 1 + .../scm/container/common/helpers/Pipeline.java | 4 + .../hadoop/hdds/scm/pipeline/Pipeline.java | 103 +- .../StorageContainerLocationProtocol.java | 2 +- ...rLocationProtocolClientSideTranslatorPB.java | 2 +- .../scm/storage/ContainerProtocolCalls.java | 7 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 + .../org/apache/hadoop/ozone/OzoneConsts.java | 5 +- ...rLocationProtocolServerSideTranslatorPB.java | 2 +- .../org/apache/hadoop/utils/RocksDBStore.java | 4 +- .../org/apache/hadoop/utils/db/RDBStore.java | 4 +- .../main/proto/DatanodeContainerProtocol.proto | 5 +- hadoop-hdds/common/src/main/proto/hdds.proto | 18 +- .../common/src/main/resources/ozone-default.xml | 8 + .../report/CommandStatusReportPublisher.java | 2 +- .../statemachine/DatanodeStateMachine.java | 1 - .../states/endpoint/VersionEndpointTask.java | 6 +- .../server/ratis/ContainerStateMachine.java | 85 +- .../server/ratis/XceiverServerRatis.java | 14 +- .../container/keyvalue/KeyValueHandler.java | 8 +- .../keyvalue/impl/BlockManagerImpl.java | 40 +- .../keyvalue/interfaces/BlockManager.java | 4 +- .../container/ozoneimpl/OzoneContainer.java | 5 + .../replication/ReplicationSupervisor.java | 137 +- .../common/report/TestReportPublisher.java | 13 +- .../keyvalue/TestBlockManagerImpl.java | 6 +- .../replication/TestReplicationSupervisor.java | 31 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 16 +- .../block/DatanodeDeletedBlockTransactions.java | 6 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 5 +- .../container/CloseContainerEventHandler.java | 31 +- .../scm/container/CloseContainerWatcher.java | 3 +- .../hdds/scm/container/ContainerManager.java | 70 +- .../hdds/scm/container/ContainerReplica.java | 197 ++ .../scm/container/ContainerReportHandler.java | 60 +- .../scm/container/ContainerStateManager.java | 242 +-- .../hdds/scm/container/SCMContainerManager.java | 566 +++--- .../replication/ReplicationManager.java | 38 +- .../scm/container/states/ContainerStateMap.java | 267 ++- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 74 +- .../scm/pipeline/PipelineActionHandler.java | 66 + .../hdds/scm/pipeline/PipelineFactory.java | 7 +- .../hdds/scm/pipeline/PipelineManager.java | 13 +- .../scm/pipeline/PipelineReportHandler.java | 104 + .../hdds/scm/pipeline/PipelineStateManager.java | 135 +- .../hdds/scm/pipeline/PipelineStateMap.java | 76 +- .../scm/pipeline/RatisPipelineProvider.java | 26 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 81 +- .../scm/pipeline/SimplePipelineProvider.java | 6 +- .../hdds/scm/server/ChillModePrecheck.java | 15 +- .../hdds/scm/server/SCMChillModeManager.java | 29 +- .../scm/server/SCMClientProtocolServer.java | 39 +- .../scm/server/SCMDatanodeProtocolServer.java | 2 +- .../scm/server/StorageContainerManager.java | 50 +- .../resources/webapps/scm/scm-overview.html | 2 +- .../src/main/resources/webapps/scm/scm.js | 4 + .../apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +- .../org/apache/hadoop/hdds/scm/TestUtils.java | 65 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 2 +- .../hdds/scm/block/TestDeletedBlockLog.java | 14 +- .../TestCloseContainerEventHandler.java | 22 +- .../container/TestContainerReportHandler.java | 66 +- .../container/TestContainerStateManager.java | 60 +- .../scm/container/TestSCMContainerManager.java | 117 +- .../replication/TestReplicationManager.java | 38 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hdds/scm/node/TestDeadNodeHandler.java | 195 +- .../scm/server/TestSCMChillModeManager.java | 2 +- .../container/TestCloseContainerWatcher.java | 12 +- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 2 +- .../hdds/scm/cli/container/ListSubcommand.java | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 85 +- .../hdfs/client/HdfsClientConfigKeys.java | 5 + .../federation/resolver/order/HashResolver.java | 10 +- .../src/main/webapps/router/federationhealth.js | 8 +- .../src/site/markdown/HDFSRouterFederation.md | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + .../block/BlockPoolTokenSecretManager.java | 14 +- .../token/block/BlockTokenSecretManager.java | 17 + .../hdfs/server/datanode/BlockSender.java | 27 +- .../hadoop/hdfs/server/datanode/DNConf.java | 2 +- .../server/datanode/LocalReplicaInPipeline.java | 57 +- .../hdfs/server/datanode/ReplicaInPipeline.java | 14 + .../AvailableSpaceVolumeChoosingPolicy.java | 2 +- .../fsdataset/impl/ReservedSpaceCalculator.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 4 +- .../server/namenode/FSImageFormatPBINode.java | 1 - .../hdfs/server/namenode/FSNamesystem.java | 3 +- .../hdfs/server/namenode/ImageServlet.java | 8 +- .../hadoop/hdfs/server/namenode/NameNode.java | 30 + .../namenode/NameNodeResourceChecker.java | 2 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 16 +- .../hdfs/server/namenode/TransferFsImage.java | 2 +- .../OfflineImageViewerPB.java | 31 +- .../src/main/resources/hdfs-default.xml | 43 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 51 + .../apache/hadoop/hdfs/TestHAAuxiliaryPort.java | 112 ++ .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 2 +- .../hdfs/client/impl/TestBlockReaderLocal.java | 10 +- .../hadoop/hdfs/protocol/TestLayoutVersion.java | 3 +- .../security/token/block/TestBlockToken.java | 7 +- .../server/datanode/SimulatedFSDataset.java | 18 + .../extdataset/ExternalReplicaInPipeline.java | 6 + .../TestOfflineImageViewer.java | 82 +- .../hadoop-mapreduce-client-app/pom.xml | 5 + .../v2/app/client/MRClientService.java | 15 +- .../mapreduce/v2/app/webapp/TestAMWebApp.java | 158 +- .../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 6 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 22 + .../src/main/resources/mapred-default.xml | 21 + hadoop-minicluster/pom.xml | 6 + .../apache/hadoop/ozone/client/BucketArgs.java | 2 +- .../ozone/client/io/ChunkGroupInputStream.java | 12 +- .../ozone/client/io/ChunkGroupOutputStream.java | 6 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 4 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 1 - .../dist/src/main/compose/ozonefs/docker-config | 2 +- .../src/main/smoketest/ozonefs/ozonefs.robot | 52 +- .../main/smoketest/s3/objectmultidelete.robot | 44 + hadoop-ozone/docs/content/OzoneFS.md | 6 +- hadoop-ozone/docs/content/Settings.md | 4 +- .../TestContainerStateManagerIntegration.java | 219 +-- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 7 +- .../hdds/scm/pipeline/TestPipelineClose.java | 10 +- .../scm/pipeline/TestPipelineStateManager.java | 209 +- .../scm/pipeline/TestRatisPipelineProvider.java | 18 +- .../scm/pipeline/TestSCMPipelineManager.java | 187 ++ .../pipeline/TestSimplePipelineProvider.java | 16 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 10 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 2 +- .../ozone/TestStorageContainerManager.java | 4 +- .../ozone/client/rest/TestOzoneRestClient.java | 4 +- .../rpc/TestCloseContainerHandlingByClient.java | 10 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 114 +- .../container/TestContainerReplication.java | 2 +- .../common/impl/TestCloseContainerHandler.java | 2 +- .../common/impl/TestContainerPersistence.java | 64 +- .../commandhandler/TestBlockDeletion.java | 3 +- .../TestCloseContainerByPipeline.java | 10 +- .../TestCloseContainerHandler.java | 4 +- .../ozone/om/TestContainerReportWithKeys.java | 2 +- .../hadoop/ozone/om/TestScmChillMode.java | 36 +- .../hadoop/ozone/ozShell/TestOzoneShell.java | 42 + .../ozone/scm/TestXceiverClientManager.java | 15 +- .../hadoop/ozone/web/client/TestKeys.java | 4 +- .../ozone/web/handlers/VolumeHandler.java | 6 +- .../hadoop/ozone/web/interfaces/Accounting.java | 2 +- .../hadoop/ozone/web/interfaces/Volume.java | 12 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 2 +- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 6 +- .../apache/hadoop/ozone/om/OzoneManager.java | 10 +- .../hadoop/ozone/om/OzoneManagerLock.java | 7 +- .../hadoop/ozone/om/ServiceListJSONServlet.java | 4 +- .../hadoop/ozone/web/ozShell/Handler.java | 6 +- .../apache/hadoop/ozone/web/ozShell/Shell.java | 5 +- .../web/ozShell/bucket/BucketCommands.java | 3 +- .../web/ozShell/bucket/S3BucketMapping.java | 99 + .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 17 +- .../fs/ozone/TestOzoneFileInterfaces.java | 2 +- .../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 121 ++ .../hadoop/ozone/s3/HeaderPreprocessor.java | 47 + .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 30 +- .../ozone/s3/endpoint/BucketEndpoint.java | 71 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 9 +- .../ozone/s3/endpoint/MultiDeleteRequest.java | 96 + .../ozone/s3/endpoint/MultiDeleteResponse.java | 154 ++ .../ozone/s3/endpoint/ObjectEndpoint.java | 20 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 39 +- .../ozone/s3/header/AuthorizationHeaderV2.java | 12 +- .../ozone/s3/header/AuthorizationHeaderV4.java | 12 +- .../hadoop/ozone/s3/header/Credential.java | 3 +- .../s3gateway/src/main/resources/browser.html | 617 ++++++ .../ozone/s3/TestVirtualHostStyleFilter.java | 67 +- .../hadoop/ozone/s3/endpoint/TestBucketGet.java | 10 +- .../ozone/s3/endpoint/TestObjectHead.java | 5 + .../s3/endpoint/TestObjectMultiDelete.java | 86 + .../ozone/s3/exception/TestOS3Exception.java | 2 +- .../GenerateOzoneRequiredConfigurations.java | 28 +- .../genesis/BenchMarkContainerStateMap.java | 14 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 8 +- .../freon/TestFreonWithDatanodeRestart.java | 122 ++ ...TestGenerateOzoneRequiredConfigurations.java | 19 + .../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +- hadoop-project/pom.xml | 9 +- .../hadoop/tools/mapred/CopyCommitter.java | 8 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 4 - hadoop-tools/hadoop-extras/pom.xml | 10 + .../hdfs/server/namenode/ImageWriter.java | 5 +- hadoop-tools/hadoop-gridmix/pom.xml | 10 + hadoop-tools/hadoop-streaming/pom.xml | 10 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 + .../hadoop-yarn/conf/yarn-env.sh | 1 + .../hadoop-yarn-services-api/pom.xml | 2 +- .../hadoop/yarn/service/webapp/ApiServer.java | 7 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 154 +- .../hadoop-yarn-services-core/pom.xml | 2 +- .../hadoop/yarn/service/ClientAMService.java | 9 +- .../api/records/ComponentContainers.java | 96 + .../yarn/service/api/records/ConfigFormat.java | 5 +- .../yarn/service/client/ServiceClient.java | 23 +- .../hadoop/yarn/service/utils/ConfigUtils.java | 56 - .../hadoop/yarn/service/utils/FilterUtils.java | 25 +- .../service/utils/PublishedConfiguration.java | 9 - .../utils/PublishedConfigurationOutputter.java | 2 - .../yarn/service/utils/ServiceApiUtil.java | 6 + .../yarn/service/client/TestServiceClient.java | 59 +- .../yarn/service/utils/TestFilterUtils.java | 39 +- .../apache/hadoop/yarn/webapp/Dispatcher.java | 12 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 25 +- .../src/main/resources/yarn-default.xml | 5 +- .../yarn/util/resource/TestResourceUtils.java | 51 +- .../dev-support/findbugs-exclude.xml | 19 + .../hadoop-yarn/hadoop-yarn-csi/pom.xml | 209 ++ .../hadoop/yarn/csi/client/CsiClient.java | 39 + .../hadoop/yarn/csi/client/CsiClientImpl.java | 51 + .../hadoop/yarn/csi/client/CsiGrpcClient.java | 127 ++ .../hadoop/yarn/csi/client/package-info.java | 21 + .../apache/hadoop/yarn/csi/package-info.java | 21 + .../hadoop/yarn/csi/utils/GrpcHelper.java | 52 + .../hadoop/yarn/csi/utils/package-info.java | 21 + .../hadoop-yarn-csi/src/main/proto/csi.proto | 1114 +++++++++++ .../hadoop/yarn/csi/client/FakeCsiDriver.java | 65 + .../yarn/csi/client/FakeCsiIdentityService.java | 42 + .../hadoop/yarn/csi/client/TestCsiClient.java | 77 + .../hadoop/yarn/csi/client/package-info.java | 21 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 235 +-- .../apache/hadoop/registry/cli/RegistryCli.java | 497 ----- .../hadoop/registry/client/api/BindFlags.java | 41 - .../registry/client/api/DNSOperations.java | 60 - .../client/api/DNSOperationsFactory.java | 78 - .../registry/client/api/RegistryConstants.java | 390 ---- .../registry/client/api/RegistryOperations.java | 182 -- .../client/api/RegistryOperationsFactory.java | 160 -- .../registry/client/api/package-info.java | 35 - .../registry/client/binding/JsonSerDeser.java | 117 -- .../client/binding/RegistryPathUtils.java | 238 --- .../client/binding/RegistryTypeUtils.java | 291 --- .../registry/client/binding/RegistryUtils.java | 399 ---- .../registry/client/binding/package-info.java | 22 - .../AuthenticationFailedException.java | 39 - .../exceptions/InvalidPathnameException.java | 40 - .../exceptions/InvalidRecordException.java | 41 - .../NoChildrenForEphemeralsException.java | 48 - .../exceptions/NoPathPermissionsException.java | 45 - .../client/exceptions/NoRecordException.java | 45 - .../client/exceptions/RegistryIOException.java | 58 - .../client/exceptions/package-info.java | 33 - .../impl/FSRegistryOperationsService.java | 249 --- .../client/impl/RegistryOperationsClient.java | 55 - .../registry/client/impl/package-info.java | 26 - .../client/impl/zk/BindingInformation.java | 41 - .../registry/client/impl/zk/CuratorService.java | 896 --------- .../registry/client/impl/zk/ListenerHandle.java | 25 - .../registry/client/impl/zk/PathListener.java | 30 - .../client/impl/zk/RegistryBindingSource.java | 36 - .../impl/zk/RegistryInternalConstants.java | 81 - .../impl/zk/RegistryOperationsService.java | 165 -- .../client/impl/zk/RegistrySecurity.java | 1143 ----------- .../registry/client/impl/zk/ZKPathDumper.java | 133 -- .../client/impl/zk/ZookeeperConfigOptions.java | 118 -- .../registry/client/impl/zk/package-info.java | 39 - .../registry/client/types/AddressTypes.java | 94 - .../hadoop/registry/client/types/Endpoint.java | 247 --- .../registry/client/types/ProtocolTypes.java | 99 - .../client/types/RegistryPathStatus.java | 123 -- .../registry/client/types/ServiceRecord.java | 312 --- .../registry/client/types/package-info.java | 41 - .../client/types/yarn/PersistencePolicies.java | 50 - .../types/yarn/YarnRegistryAttributes.java | 42 - .../dns/ApplicationServiceRecordProcessor.java | 366 ---- .../server/dns/BaseServiceRecordProcessor.java | 470 ----- .../dns/ContainerServiceRecordProcessor.java | 284 --- .../hadoop/registry/server/dns/LookupTask.java | 39 - .../dns/PrivilegedRegistryDNSStarter.java | 80 - .../server/dns/RecordCreatorFactory.java | 275 --- .../hadoop/registry/server/dns/RegistryDNS.java | 1800 ------------------ .../registry/server/dns/RegistryDNSServer.java | 267 --- .../registry/server/dns/ReverseZoneUtils.java | 171 -- .../registry/server/dns/SecureableZone.java | 151 -- .../server/dns/ServiceRecordProcessor.java | 53 - .../registry/server/dns/ZoneSelector.java | 33 - .../registry/server/dns/package-info.java | 26 - .../integration/SelectByYarnPersistence.java | 60 - .../server/integration/package-info.java | 23 - .../hadoop/registry/server/package-info.java | 27 - .../server/services/AddingCompositeService.java | 56 - .../services/DeleteCompletionCallback.java | 57 - .../server/services/MicroZookeeperService.java | 282 --- .../services/MicroZookeeperServiceKeys.java | 69 - .../server/services/RegistryAdminService.java | 529 ----- .../registry/server/services/package-info.java | 39 - .../src/main/resources/.keep | 0 .../src/main/tla/yarnregistry.tla | 582 ------ .../hadoop/registry/AbstractRegistryTest.java | 118 -- .../hadoop/registry/AbstractZKRegistryTest.java | 113 -- .../hadoop/registry/RegistryTestHelper.java | 353 ---- .../hadoop/registry/cli/TestRegistryCli.java | 197 -- .../client/binding/TestMarshalling.java | 143 -- .../binding/TestRegistryOperationUtils.java | 56 - .../client/binding/TestRegistryPathUtils.java | 186 -- .../client/impl/CuratorEventCatcher.java | 68 - .../client/impl/TestCuratorService.java | 249 --- .../impl/TestFSRegistryOperationsService.java | 298 --- .../client/impl/TestMicroZookeeperService.java | 60 - .../integration/TestYarnPolicySelector.java | 65 - .../operations/TestRegistryOperations.java | 331 ---- .../secure/AbstractSecureRegistryTest.java | 368 ---- .../registry/secure/KerberosConfiguration.java | 100 - .../secure/TestRegistrySecurityHelper.java | 211 -- .../registry/secure/TestSecureLogins.java | 231 --- .../registry/secure/TestSecureRegistry.java | 213 --- .../registry/server/dns/TestRegistryDNS.java | 725 ------- .../server/dns/TestReverseZoneUtils.java | 89 - .../server/dns/TestSecureRegistryDNS.java | 44 - .../test/resources/0.17.172.in-addr.arpa.zone | 36 - .../src/test/resources/log4j.properties | 63 - .../src/test/resources/test.private | 32 - .../hadoop-yarn-server-common/pom.xml | 2 +- .../hadoop-yarn-server-nodemanager/pom.xml | 2 +- .../containermanager/ContainerManagerImpl.java | 55 +- .../container/ContainerImpl.java | 15 +- .../container/SlidingWindowRetryPolicy.java | 4 + .../runtime/DockerLinuxContainerRuntime.java | 3 +- .../fpga/IntelFpgaOpenclPlugin.java | 4 +- .../runtime/ContainerRuntimeConstants.java | 4 + .../test/test-container-executor.c | 54 +- .../nodemanager/DummyContainerManager.java | 3 +- .../nodemanager/TestNodeManagerResync.java | 3 +- .../BaseContainerManagerTest.java | 10 +- .../containermanager/TestContainerManager.java | 3 +- .../TestContainerManagerRecovery.java | 3 +- .../container/TestContainer.java | 3 +- .../container/TestSlidingWindowRetryPolicy.java | 26 +- .../resources/gpu/TestGpuResourceHandler.java | 3 +- .../TestDelegatingLinuxContainerRuntime.java | 5 +- .../runtime/TestDockerContainerRuntime.java | 12 +- .../resourceplugin/fpga/TestFpgaDiscoverer.java | 64 +- .../server/resourcemanager/ResourceManager.java | 3 + .../recovery/FileSystemRMStateStore.java | 53 + .../recovery/LeveldbRMStateStore.java | 62 + .../recovery/MemoryRMStateStore.java | 18 + .../recovery/NullRMStateStore.java | 9 + .../resourcemanager/recovery/RMStateStore.java | 100 +- .../recovery/RMStateStoreEventType.java | 1 + .../recovery/RMStateStoreProxyCAEvent.java | 49 + .../recovery/ZKRMStateStore.java | 60 +- .../CapacitySchedulerConfiguration.java | 5 +- .../security/ProxyCAManager.java | 23 +- .../resourcemanager/TestClientRMService.java | 17 +- .../recovery/RMStateStoreTestBase.java | 33 + .../recovery/TestFSRMStateStore.java | 1 + .../recovery/TestLeveldbRMStateStore.java | 6 + .../recovery/TestZKRMStateStore.java | 1 + .../capacity/TestCapacityScheduler.java | 4 +- .../security/TestProxyCAManager.java | 54 + .../security/TestRMDelegationTokens.java | 12 +- .../collector/TimelineCollectorManager.java | 2 +- .../hadoop/yarn/server/webproxy/ProxyCA.java | 33 + .../server/webproxy/WebAppProxyServlet.java | 1 + .../yarn/server/webproxy/TestProxyCA.java | 109 ++ .../src/site/markdown/CapacityScheduler.md | 14 +- .../src/site/markdown/NodeManager.md | 10 +- .../src/site/markdown/UsingFPGA.md | 2 +- .../site/markdown/YarnApplicationSecurity.md | 84 +- .../src/site/markdown/registry/index.md | 30 - .../markdown/registry/registry-configuration.md | 397 ---- .../site/markdown/registry/registry-security.md | 120 -- .../registry/using-the-yarn-service-registry.md | 273 --- .../src/site/markdown/registry/yarn-registry.md | 1018 ---------- .../src/site/markdown/yarn-service/Examples.md | 2 +- .../src/site/markdown/yarn-service/Overview.md | 2 +- .../site/markdown/yarn-service/RegistryDNS.md | 200 -- .../markdown/yarn-service/ServiceDiscovery.md | 83 +- .../markdown/yarn-service/YarnServiceAPI.md | 2 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 1 + start-build-env.sh | 2 +- 518 files changed, 29289 insertions(+), 22374 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efb79f4e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
