Merge branch 'trunk' into HDDS-4 Conflicts: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7119be30 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7119be30 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7119be30 Branch: refs/heads/HDDS-4 Commit: 7119be30bd570e65c97ff88e8aa84705c79f227b Parents: 2115256 f3f5e7a Author: Anu Engineer <aengin...@apache.org> Authored: Mon Nov 5 12:37:32 2018 -0800 Committer: Anu Engineer <aengin...@apache.org> Committed: Mon Nov 5 12:37:32 2018 -0800 ---------------------------------------------------------------------- LICENSE.txt | 5 +- NOTICE.txt | 8 +- dev-support/bin/dist-layout-stitching | 1 + dev-support/docker/Dockerfile | 1 + .../assemblies/hadoop-registry-dist.xml | 41 + .../hadoop-client-minicluster/pom.xml | 2 +- .../hadoop-common/src/main/bin/hadoop | 6 + .../hadoop-common/src/main/conf/hadoop-env.sh | 13 + .../fs/CommonConfigurationKeysPublic.java | 31 +- .../io/compress/zstd/ZStandardDecompressor.java | 4 +- .../main/java/org/apache/hadoop/ipc/Server.java | 114 +- .../org/apache/hadoop/security/Credentials.java | 15 + .../hadoop/security/SaslPropertiesResolver.java | 4 +- .../hadoop/security/UserGroupInformation.java | 192 +- .../hadoop/security/token/DtFileOperations.java | 28 +- .../hadoop/security/token/DtUtilShell.java | 37 +- .../io/compress/zstd/ZStandardCompressor.c | 11 +- .../io/compress/zstd/ZStandardDecompressor.c | 1 + .../src/main/resources/core-default.xml | 8 + .../src/site/markdown/CommandsManual.md | 1 + .../src/site/markdown/CredentialProviderAPI.md | 130 +- .../site/markdown/registry/hadoop-registry.md | 1018 ++++++++++ .../src/site/markdown/registry/index.md | 31 + .../markdown/registry/registry-configuration.md | 397 ++++ .../src/site/markdown/registry/registry-dns.md | 224 +++ .../site/markdown/registry/registry-security.md | 120 ++ .../using-the-hadoop-service-registry.md | 273 +++ .../hadoop/crypto/key/TestKeyProvider.java | 32 +- .../TestZStandardCompressorDecompressor.java | 10 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 53 +- .../apache/hadoop/security/TestCredentials.java | 57 +- .../hadoop/security/TestUGILoginFromKeytab.java | 56 + .../security/TestUserGroupInformation.java | 2 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 105 + .../hadoop/security/token/TestDtUtilShell.java | 44 + .../dev-support/findbugs-exclude.xml | 33 + hadoop-common-project/hadoop-registry/pom.xml | 309 +++ .../apache/hadoop/registry/cli/RegistryCli.java | 497 +++++ .../hadoop/registry/client/api/BindFlags.java | 41 + .../registry/client/api/DNSOperations.java | 60 + .../client/api/DNSOperationsFactory.java | 78 + .../registry/client/api/RegistryConstants.java | 388 ++++ .../registry/client/api/RegistryOperations.java | 182 ++ .../client/api/RegistryOperationsFactory.java | 160 ++ .../registry/client/api/package-info.java | 35 + .../registry/client/binding/JsonSerDeser.java | 117 ++ .../client/binding/RegistryPathUtils.java | 238 +++ .../client/binding/RegistryTypeUtils.java | 291 +++ .../registry/client/binding/RegistryUtils.java | 399 ++++ .../registry/client/binding/package-info.java | 22 + .../AuthenticationFailedException.java | 39 + .../exceptions/InvalidPathnameException.java | 40 + .../exceptions/InvalidRecordException.java | 41 + .../NoChildrenForEphemeralsException.java | 48 + .../exceptions/NoPathPermissionsException.java | 45 + .../client/exceptions/NoRecordException.java | 45 + .../client/exceptions/RegistryIOException.java | 58 + .../client/exceptions/package-info.java | 33 + .../impl/FSRegistryOperationsService.java | 248 +++ .../client/impl/RegistryOperationsClient.java | 55 + .../registry/client/impl/package-info.java | 26 + .../client/impl/zk/BindingInformation.java | 41 + .../registry/client/impl/zk/CuratorService.java | 896 +++++++++ .../registry/client/impl/zk/ListenerHandle.java | 25 + .../registry/client/impl/zk/PathListener.java | 30 + .../client/impl/zk/RegistryBindingSource.java | 36 + .../impl/zk/RegistryInternalConstants.java | 81 + .../impl/zk/RegistryOperationsService.java | 165 ++ .../client/impl/zk/RegistrySecurity.java | 1143 +++++++++++ .../registry/client/impl/zk/ZKPathDumper.java | 133 ++ .../client/impl/zk/ZookeeperConfigOptions.java | 118 ++ .../registry/client/impl/zk/package-info.java | 39 + .../registry/client/types/AddressTypes.java | 94 + .../hadoop/registry/client/types/Endpoint.java | 247 +++ .../registry/client/types/ProtocolTypes.java | 99 + .../client/types/RegistryPathStatus.java | 123 ++ .../registry/client/types/ServiceRecord.java | 312 +++ .../registry/client/types/package-info.java | 41 + .../client/types/yarn/PersistencePolicies.java | 50 + .../types/yarn/YarnRegistryAttributes.java | 42 + .../registry/conf/RegistryConfiguration.java | 41 + .../hadoop/registry/conf/package-info.java | 22 + .../dns/ApplicationServiceRecordProcessor.java | 366 ++++ .../server/dns/BaseServiceRecordProcessor.java | 470 +++++ .../dns/ContainerServiceRecordProcessor.java | 284 +++ .../hadoop/registry/server/dns/LookupTask.java | 39 + .../dns/PrivilegedRegistryDNSStarter.java | 81 + .../server/dns/RecordCreatorFactory.java | 275 +++ .../hadoop/registry/server/dns/RegistryDNS.java | 1800 ++++++++++++++++++ .../registry/server/dns/RegistryDNSServer.java | 267 +++ .../registry/server/dns/ReverseZoneUtils.java | 171 ++ .../registry/server/dns/SecureableZone.java | 151 ++ .../server/dns/ServiceRecordProcessor.java | 53 + .../registry/server/dns/ZoneSelector.java | 33 + .../registry/server/dns/package-info.java | 26 + .../integration/SelectByYarnPersistence.java | 60 + .../server/integration/package-info.java | 23 + .../hadoop/registry/server/package-info.java | 27 + .../server/services/AddingCompositeService.java | 56 + .../services/DeleteCompletionCallback.java | 57 + .../server/services/MicroZookeeperService.java | 282 +++ .../services/MicroZookeeperServiceKeys.java | 69 + .../server/services/RegistryAdminService.java | 529 +++++ .../registry/server/services/package-info.java | 39 + .../src/main/tla/hadoopregistry.tla | 582 ++++++ .../hadoop/registry/AbstractRegistryTest.java | 118 ++ .../hadoop/registry/AbstractZKRegistryTest.java | 113 ++ .../hadoop/registry/RegistryTestHelper.java | 353 ++++ .../hadoop/registry/cli/TestRegistryCli.java | 197 ++ .../client/binding/TestMarshalling.java | 143 ++ .../binding/TestRegistryOperationUtils.java | 56 + .../client/binding/TestRegistryPathUtils.java | 186 ++ .../client/impl/CuratorEventCatcher.java | 68 + .../client/impl/TestCuratorService.java | 249 +++ .../impl/TestFSRegistryOperationsService.java | 298 +++ .../client/impl/TestMicroZookeeperService.java | 61 + .../integration/TestYarnPolicySelector.java | 65 + .../operations/TestRegistryOperations.java | 331 ++++ .../secure/AbstractSecureRegistryTest.java | 369 ++++ .../registry/secure/KerberosConfiguration.java | 100 + .../secure/TestRegistrySecurityHelper.java | 211 ++ .../registry/secure/TestSecureLogins.java | 234 +++ .../registry/secure/TestSecureRegistry.java | 213 +++ .../registry/server/dns/TestRegistryDNS.java | 725 +++++++ .../server/dns/TestReverseZoneUtils.java | 89 + .../server/dns/TestSecureRegistryDNS.java | 44 + .../test/resources/0.17.172.in-addr.arpa.zone | 36 + .../src/test/resources/log4j.properties | 63 + .../src/test/resources/test.private | 32 + hadoop-common-project/pom.xml | 1 + .../hadoop/hdds/scm/XceiverClientGrpc.java | 197 +- .../hadoop/hdds/scm/XceiverClientManager.java | 20 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 37 +- .../scm/client/ContainerOperationClient.java | 46 +- .../hdds/scm/storage/ChunkOutputStream.java | 17 +- .../common/src/main/conf/log4j.properties | 157 -- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 125 +- .../org/apache/hadoop/hdds/cli/GenericCli.java | 5 +- .../hadoop/hdds/cli/GenericParentCommand.java | 4 + .../org/apache/hadoop/hdds/client/BlockID.java | 85 +- .../hadoop/hdds/client/ContainerBlockID.java | 79 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 15 + .../hadoop/hdds/scm/XceiverClientSpi.java | 2 +- .../hadoop/hdds/scm/client/ScmClient.java | 8 +- .../hdds/scm/container/ContainerInfo.java | 10 +- .../common/helpers/AllocatedBlock.java | 22 +- .../common/helpers/ContainerWithPipeline.java | 3 +- .../scm/container/common/helpers/Pipeline.java | 315 --- .../container/common/helpers/PipelineID.java | 97 - .../hadoop/hdds/scm/pipeline/Pipeline.java | 41 +- .../scm/pipeline/PipelineNotFoundException.java | 46 + .../StorageContainerLocationProtocol.java | 2 +- ...kLocationProtocolClientSideTranslatorPB.java | 9 +- ...rLocationProtocolClientSideTranslatorPB.java | 6 +- .../scm/storage/ContainerProtocolCalls.java | 30 +- .../hdds/security/x509/SecurityConfig.java | 2 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 16 +- .../apache/hadoop/ozone/common/BlockGroup.java | 3 +- .../container/common/helpers/BlockData.java | 8 +- ...kLocationProtocolServerSideTranslatorPB.java | 2 +- .../org/apache/hadoop/utils/RocksDBStore.java | 4 +- .../org/apache/hadoop/utils/db/RDBStore.java | 4 +- .../main/java/org/apache/ratis/RatisHelper.java | 15 +- .../main/proto/DatanodeContainerProtocol.proto | 34 +- .../main/proto/ScmBlockLocationProtocol.proto | 2 +- .../StorageContainerLocationProtocol.proto | 4 +- hadoop-hdds/common/src/main/proto/hdds.proto | 11 +- .../common/src/main/resources/ozone-default.xml | 58 +- .../TestCertificateSignRequest.java | 2 +- .../x509/certificates/TestRootCertificate.java | 2 +- .../x509/keys/TestHDDSKeyGenerator.java | 2 +- .../apache/hadoop/hdds/scm/HddsServerUtil.java | 53 +- .../container/common/impl/ContainerData.java | 22 +- .../common/impl/ContainerDataYaml.java | 6 +- .../container/common/impl/HddsDispatcher.java | 13 +- .../container/common/interfaces/Container.java | 9 +- .../statemachine/DatanodeStateMachine.java | 4 +- .../states/endpoint/VersionEndpointTask.java | 79 +- .../transport/server/XceiverServerGrpc.java | 2 +- .../transport/server/ratis/CSMMetrics.java | 5 +- .../server/ratis/ContainerStateMachine.java | 216 ++- .../server/ratis/XceiverServerRatis.java | 58 +- .../container/common/volume/VolumeInfo.java | 19 +- .../container/common/volume/VolumeSet.java | 11 +- .../container/keyvalue/KeyValueContainer.java | 31 +- .../keyvalue/KeyValueContainerData.java | 9 +- .../container/keyvalue/KeyValueHandler.java | 28 +- .../container/keyvalue/helpers/BlockUtils.java | 4 +- .../container/keyvalue/helpers/ChunkUtils.java | 30 +- .../keyvalue/helpers/SmallFileUtils.java | 7 + .../keyvalue/impl/BlockManagerImpl.java | 38 +- .../keyvalue/impl/ChunkManagerImpl.java | 5 +- .../keyvalue/interfaces/BlockManager.java | 3 +- .../keyvalue/interfaces/ChunkManager.java | 3 +- .../container/ozoneimpl/OzoneContainer.java | 5 + .../replication/ReplicationSupervisor.java | 137 +- .../commands/CloseContainerCommand.java | 2 +- .../StorageContainerDatanodeProtocol.proto | 57 +- .../ozone/container/common/SCMTestUtils.java | 3 +- .../ozone/container/common/ScmTestMock.java | 14 +- .../common/TestDatanodeStateMachine.java | 3 +- .../common/TestKeyValueContainerData.java | 6 +- .../common/impl/TestContainerDataYaml.java | 8 +- .../container/common/impl/TestContainerSet.java | 16 +- .../container/common/volume/TestHddsVolume.java | 9 +- .../container/common/volume/TestVolumeSet.java | 4 +- .../keyvalue/TestChunkManagerImpl.java | 37 +- .../keyvalue/TestKeyValueContainer.java | 15 +- .../container/keyvalue/TestKeyValueHandler.java | 2 +- .../container/ozoneimpl/TestOzoneContainer.java | 4 +- .../replication/TestReplicationSupervisor.java | 31 +- .../hadoop/hdds/server/BaseHttpServer.java | 8 +- .../apache/hadoop/hdds/server/ServerUtils.java | 49 +- hadoop-hdds/pom.xml | 161 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 9 +- .../block/DatanodeDeletedBlockTransactions.java | 4 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 15 +- .../container/CloseContainerEventHandler.java | 4 +- .../hdds/scm/container/ContainerManager.java | 6 +- .../scm/container/ContainerReportHandler.java | 2 +- .../scm/container/ContainerStateManager.java | 25 +- .../hdds/scm/container/SCMContainerManager.java | 92 +- .../hadoop/hdds/scm/events/SCMEvents.java | 9 - .../hadoop/hdds/scm/node/NodeManager.java | 4 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 4 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 4 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 30 +- .../hdds/scm/node/states/Node2PipelineMap.java | 8 +- .../hdds/scm/pipeline/PipelineFactory.java | 6 +- .../hdds/scm/pipeline/PipelineManager.java | 10 +- .../hdds/scm/pipeline/PipelineProvider.java | 2 +- .../scm/pipeline/PipelineReportHandler.java | 16 +- .../hdds/scm/pipeline/PipelineStateManager.java | 27 +- .../hdds/scm/pipeline/PipelineStateMap.java | 91 +- .../scm/pipeline/RatisPipelineProvider.java | 15 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 43 +- .../scm/pipeline/SimplePipelineProvider.java | 13 +- .../pipelines/PipelineActionEventHandler.java | 62 - .../scm/pipelines/PipelineCloseHandler.java | 52 - .../hdds/scm/pipelines/PipelineManager.java | 171 -- .../scm/pipelines/PipelineReportHandler.java | 59 - .../hdds/scm/pipelines/PipelineSelector.java | 481 ----- .../scm/pipelines/PipelineStateManager.java | 136 -- .../hadoop/hdds/scm/pipelines/package-info.java | 38 - .../scm/pipelines/ratis/RatisManagerImpl.java | 129 -- .../hdds/scm/pipelines/ratis/package-info.java | 18 - .../standalone/StandaloneManagerImpl.java | 122 -- .../scm/pipelines/standalone/package-info.java | 18 - .../scm/server/SCMClientProtocolServer.java | 3 +- .../hadoop/hdds/scm/server/SCMStorage.java | 4 +- .../scm/server/StorageContainerManager.java | 48 +- .../apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +- .../hadoop/hdds/scm/HddsWhiteboxTestUtils.java | 103 + .../hadoop/hdds/scm/TestHddsServerUtils.java | 203 ++ .../org/apache/hadoop/hdds/scm/TestUtils.java | 20 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 13 +- .../hdds/scm/block/TestDeletedBlockLog.java | 36 +- .../hdds/scm/container/MockNodeManager.java | 4 +- .../TestCloseContainerEventHandler.java | 24 +- .../container/TestContainerReportHandler.java | 17 +- .../container/TestContainerStateManager.java | 29 +- .../scm/container/TestSCMContainerManager.java | 50 +- .../replication/TestReplicationManager.java | 29 +- .../hdds/scm/node/TestContainerPlacement.java | 12 +- .../hdds/scm/node/TestDeadNodeHandler.java | 11 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 6 +- .../ozone/container/common/TestEndPoint.java | 5 +- .../testutils/ReplicationNodeManagerMock.java | 4 +- .../hdds/scm/cli/container/InfoSubcommand.java | 13 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 12 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 85 +- .../hdfs/client/HdfsClientConfigKeys.java | 5 + .../hdfs/protocol/SnapshotDiffReport.java | 65 + .../http/server/HttpFSParametersProvider.java | 10 +- .../fs/http/client/BaseTestHttpFSWith.java | 60 + .../hadoop/fs/http/server/TestHttpFSServer.java | 65 +- .../http/server/TestHttpFSServerWebServer.java | 11 + .../org/apache/hadoop/test/TestHdfsHelper.java | 24 +- .../src/CMakeLists.txt | 22 +- .../src/main/native/libhdfspp/CMakeLists.txt | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 12 +- .../hadoop/hdfs/protocol/BlockListAsLongs.java | 2 +- .../QJournalProtocolServerSideTranslatorPB.java | 2 +- .../block/BlockPoolTokenSecretManager.java | 34 +- .../token/block/BlockTokenSecretManager.java | 39 +- .../hadoop/hdfs/server/balancer/Balancer.java | 15 +- .../server/blockmanagement/BlockManager.java | 39 +- .../blockmanagement/BlockPlacementPolicy.java | 1 - .../CombinedHostFileManager.java | 6 +- .../blockmanagement/CorruptReplicasMap.java | 2 +- .../blockmanagement/DatanodeAdminManager.java | 8 +- .../blockmanagement/DatanodeDescriptor.java | 4 +- .../blockmanagement/HeartbeatManager.java | 2 +- .../server/blockmanagement/HostFileManager.java | 7 +- .../hdfs/server/blockmanagement/HostSet.java | 8 +- .../server/blockmanagement/SlowPeerTracker.java | 5 +- .../server/datanode/BlockPoolSliceStorage.java | 60 +- .../server/datanode/BlockRecoveryWorker.java | 15 +- .../hdfs/server/datanode/BlockScanner.java | 6 +- .../hadoop/hdfs/server/datanode/DataNode.java | 10 +- .../hdfs/server/datanode/DataStorage.java | 4 +- .../hdfs/server/datanode/DirectoryScanner.java | 1 - .../hdfs/server/datanode/FileIoProvider.java | 3 - .../hdfs/server/datanode/VolumeScanner.java | 4 +- .../server/datanode/checker/AbstractFuture.java | 13 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 12 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 13 +- .../datanode/metrics/OutlierDetector.java | 3 +- .../diskbalancer/DiskBalancerException.java | 1 - .../datamodel/DiskBalancerCluster.java | 11 +- .../datamodel/DiskBalancerDataNode.java | 10 +- .../diskbalancer/planner/GreedyPlanner.java | 2 +- .../hadoop/hdfs/server/namenode/AclStorage.java | 18 +- .../server/namenode/EncryptionZoneManager.java | 42 +- .../hdfs/server/namenode/FSDirectory.java | 8 +- .../hdfs/server/namenode/FSNamesystem.java | 44 +- .../hadoop/hdfs/server/namenode/INode.java | 4 +- .../hdfs/server/namenode/INodeReference.java | 6 +- .../hdfs/server/namenode/INodesInPath.java | 4 +- .../hdfs/server/namenode/JournalManager.java | 2 +- .../hdfs/server/namenode/LeaseManager.java | 2 +- .../server/namenode/MetaRecoveryContext.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 36 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 16 +- .../hdfs/server/namenode/NamenodeFsck.java | 9 +- .../hadoop/hdfs/server/namenode/Quota.java | 5 +- .../server/namenode/ReencryptionHandler.java | 2 +- .../server/namenode/XAttrPermissionFilter.java | 4 +- .../hdfs/server/namenode/XAttrStorage.java | 8 +- .../snapshot/AbstractINodeDiffList.java | 8 +- .../namenode/snapshot/DiffListBySkipList.java | 9 +- .../snapshot/DirectorySnapshottableFeature.java | 5 + .../namenode/snapshot/SnapshotDiffInfo.java | 50 +- .../sps/BlockStorageMovementNeeded.java | 5 +- .../namenode/sps/DatanodeCacheManager.java | 2 +- .../sps/StoragePolicySatisfyManager.java | 14 +- .../startupprogress/StartupProgressView.java | 4 +- .../server/namenode/top/metrics/TopMetrics.java | 17 +- .../namenode/top/window/RollingWindow.java | 18 +- .../top/window/RollingWindowManager.java | 2 +- .../protocol/BlockStorageMovementCommand.java | 11 +- .../hdfs/server/protocol/DatanodeProtocol.java | 2 +- .../hdfs/server/protocol/NamenodeProtocol.java | 5 +- .../sps/ExternalSPSBlockMoveTaskHandler.java | 2 + .../org/apache/hadoop/hdfs/tools/DFSck.java | 13 +- .../offlineEditsViewer/OfflineEditsViewer.java | 4 +- .../offlineEditsViewer/OfflineEditsVisitor.java | 2 +- .../StatisticsEditsVisitor.java | 4 +- .../NameDistributionVisitor.java | 4 +- .../OfflineImageViewerPB.java | 31 +- .../offlineImageViewer/PBImageTextWriter.java | 4 +- .../java/org/apache/hadoop/hdfs/util/Diff.java | 16 +- .../org/apache/hadoop/hdfs/util/XMLUtils.java | 4 +- .../src/main/resources/hdfs-default.xml | 11 + .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 51 + .../hadoop/hdfs/TestDFSStripedOutputStream.java | 36 +- .../apache/hadoop/hdfs/TestHAAuxiliaryPort.java | 112 ++ .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 2 +- .../security/token/block/TestBlockToken.java | 13 +- .../TestPendingReconstruction.java | 2 + ...constructStripedBlocksWithRackAwareness.java | 11 +- .../server/datanode/TestDataNodeLifeline.java | 5 + .../fsdataset/impl/TestLazyPersistFiles.java | 15 +- .../TestUpgradeDomainBlockPlacementPolicy.java | 22 +- .../TestOfflineImageViewer.java | 101 +- .../hadoop-mapreduce-client-app/pom.xml | 5 + .../v2/app/client/MRClientService.java | 15 +- .../v2/app/rm/RMContainerAllocator.java | 14 +- .../v2/app/rm/TestRMContainerAllocator.java | 14 + .../mapreduce/v2/app/webapp/TestAMWebApp.java | 158 +- .../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 6 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 22 + .../src/main/resources/mapred-default.xml | 21 + hadoop-maven-plugins/pom.xml | 4 + .../ozone/client/io/ChunkGroupInputStream.java | 12 +- .../ozone/client/io/ChunkGroupOutputStream.java | 31 +- .../ozone/client/TestHddsClientUtils.java | 137 +- .../src/main/conf/om-audit-log4j2.properties | 90 - .../common/src/main/conf/ozone-site.xml | 24 - .../java/org/apache/hadoop/ozone/OmUtils.java | 50 +- .../apache/hadoop/ozone/om/OMConfigKeys.java | 3 + .../ozone/om/helpers/OmKeyLocationInfo.java | 19 +- .../src/main/proto/OzoneManagerProtocol.proto | 1 - .../org/apache/hadoop/ozone/TestOmUtils.java | 91 + .../dist/dev-support/bin/dist-layout-stitching | 4 +- .../dist/src/main/conf/log4j.properties | 157 ++ .../src/main/conf/om-audit-log4j2.properties | 90 + hadoop-ozone/dist/src/main/conf/ozone-site.xml | 24 + .../dist/src/main/smoketest/s3/README.md | 2 +- .../dist/src/main/smoketest/s3/awss3.robot | 4 +- .../dist/src/main/smoketest/s3/objectcopy.robot | 66 + .../src/main/smoketest/s3/objectdelete.robot | 6 +- .../main/smoketest/s3/objectmultidelete.robot | 6 +- .../src/main/smoketest/s3/objectputget.robot | 2 +- hadoop-ozone/docs/content/S3.md | 130 ++ .../ozonedoc/layouts/partials/navbar.html | 5 +- .../ozonedoc/layouts/partials/sidebar.html | 8 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 29 +- .../hdds/scm/pipeline/TestNodeFailure.java | 27 +- .../hdds/scm/pipeline/TestPipelineClose.java | 41 +- .../scm/pipeline/TestPipelineStateManager.java | 171 +- .../scm/pipeline/TestRatisPipelineProvider.java | 13 +- .../scm/pipeline/TestSCMPipelineManager.java | 49 +- .../hdds/scm/pipeline/TestSCMRestart.java | 23 +- .../pipeline/TestSimplePipelineProvider.java | 13 +- .../apache/hadoop/ozone/MiniOzoneCluster.java | 16 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 41 +- .../apache/hadoop/ozone/RatisTestHelper.java | 2 +- .../TestContainerStateMachineIdempotency.java | 2 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 82 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 28 +- .../ozone/TestStorageContainerManager.java | 13 +- .../TestStorageContainerManagerHelper.java | 5 +- .../ozone/client/rest/TestOzoneRestClient.java | 4 +- .../hadoop/ozone/client/rpc/TestBCSID.java | 2 +- .../rpc/TestCloseContainerHandlingByClient.java | 72 +- .../rpc/TestContainerStateMachineFailures.java | 23 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 112 +- .../ozone/container/ContainerTestHelper.java | 76 +- .../container/TestContainerReplication.java | 6 +- .../common/impl/TestCloseContainerHandler.java | 10 +- .../common/impl/TestContainerPersistence.java | 92 +- .../commandhandler/TestBlockDeletion.java | 14 +- .../TestCloseContainerByPipeline.java | 8 +- .../TestCloseContainerHandler.java | 2 +- .../transport/server/ratis/TestCSMMetrics.java | 14 +- .../container/metrics/TestContainerMetrics.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 9 +- .../ozoneimpl/TestOzoneContainerRatis.java | 6 +- .../container/server/TestContainerServer.java | 21 +- .../apache/hadoop/ozone/om/TestOmMetrics.java | 23 +- .../hadoop/ozone/om/TestOzoneManager.java | 3 +- .../hadoop/ozone/ozShell/TestOzoneShell.java | 77 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 2 +- .../ozone/scm/TestContainerSmallFile.java | 51 + .../TestGetCommittedBlockLengthAndPutKey.java | 18 +- .../ozone/scm/TestXceiverClientManager.java | 15 +- .../hadoop/ozone/web/client/TestKeys.java | 6 +- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +- .../org/apache/hadoop/ozone/om/OMStorage.java | 5 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +- .../apache/hadoop/ozone/om/OzoneManager.java | 10 +- .../hadoop/ozone/web/ozShell/Handler.java | 139 +- .../hadoop/ozone/web/ozShell/OzoneAddress.java | 251 +++ .../web/ozShell/bucket/BucketCommands.java | 6 + .../web/ozShell/bucket/CreateBucketHandler.java | 27 +- .../web/ozShell/bucket/DeleteBucketHandler.java | 20 +- .../web/ozShell/bucket/InfoBucketHandler.java | 28 +- .../web/ozShell/bucket/ListBucketHandler.java | 26 +- .../web/ozShell/bucket/S3BucketMapping.java | 55 +- .../web/ozShell/bucket/UpdateBucketHandler.java | 20 +- .../web/ozShell/keys/DeleteKeyHandler.java | 23 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 18 +- .../ozone/web/ozShell/keys/InfoKeyHandler.java | 28 +- .../ozone/web/ozShell/keys/KeyCommands.java | 6 + .../ozone/web/ozShell/keys/ListKeyHandler.java | 28 +- .../ozone/web/ozShell/keys/PutKeyHandler.java | 21 +- .../web/ozShell/volume/CreateVolumeHandler.java | 28 +- .../web/ozShell/volume/DeleteVolumeHandler.java | 8 +- .../web/ozShell/volume/InfoVolumeHandler.java | 8 +- .../web/ozShell/volume/ListVolumeHandler.java | 27 +- .../web/ozShell/volume/UpdateVolumeHandler.java | 8 +- .../web/ozShell/volume/VolumeCommands.java | 6 + .../ozone/om/ScmBlockLocationTestIngClient.java | 22 +- .../ozone/web/ozShell/TestOzoneAddress.java | 100 + .../hadoop/ozone/web/ozShell/package-info.java | 21 + .../ITestOzoneContractGetFileStatus.java | 6 +- hadoop-ozone/pom.xml | 139 +- hadoop-ozone/s3gateway/pom.xml | 8 +- .../ozone/s3/SignedChunksInputStream.java | 99 + .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 22 + .../ozone/s3/endpoint/BucketEndpoint.java | 134 +- .../ozone/s3/endpoint/CopyObjectResponse.java | 63 + .../hadoop/ozone/s3/endpoint/EndpointBase.java | 49 +- .../ozone/s3/endpoint/ListObjectResponse.java | 22 + .../ozone/s3/endpoint/MultiDeleteRequest.java | 2 +- .../MultiDeleteRequestUnmarshaller.java | 84 + .../ozone/s3/endpoint/ObjectEndpoint.java | 185 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 6 +- .../ozone/s3/endpoint/XmlNamespaceFilter.java | 54 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 45 +- .../s3/header/AuthenticationHeaderParser.java | 61 + .../ozone/s3/header/AuthorizationHeaderV2.java | 12 +- .../ozone/s3/header/AuthorizationHeaderV4.java | 12 +- .../hadoop/ozone/s3/header/Credential.java | 3 +- .../apache/hadoop/ozone/s3/util/S3Consts.java | 38 + .../hadoop/ozone/s3/util/S3StorageType.java | 55 + .../apache/hadoop/ozone/s3/util/S3utils.java | 91 + .../hadoop/ozone/s3/util/package-info.java | 22 + .../s3gateway/src/main/resources/browser.html | 4 +- .../resources/webapps/s3gateway/WEB-INF/web.xml | 4 + .../resources/webapps/static/images/ozone.ico | Bin 0 -> 1150 bytes .../hadoop/ozone/client/OzoneBucketStub.java | 7 +- .../ozone/s3/TestSignedChunksInputStream.java | 84 + .../ozone/s3/TestVirtualHostStyleFilter.java | 20 +- .../hadoop/ozone/s3/endpoint/TestBucketGet.java | 227 ++- .../TestMultiDeleteRequestUnmarshaller.java | 76 + .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 4 +- .../s3/endpoint/TestObjectMultiDelete.java | 61 +- .../hadoop/ozone/s3/endpoint/TestPutObject.java | 164 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 22 +- .../ozone/s3/exception/TestOS3Exception.java | 2 +- .../genesis/BenchMarkContainerStateMap.java | 27 +- .../genesis/BenchMarkDatanodeDispatcher.java | 6 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 2 +- .../freon/TestFreonWithDatanodeFastRestart.java | 130 ++ .../freon/TestFreonWithDatanodeRestart.java | 103 + .../apache/hadoop/ozone/om/TestOmSQLCli.java | 3 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 10 +- hadoop-project/pom.xml | 236 +-- .../site/markdown/tools/hadoop-aliyun/index.md | 36 + .../azurebfs/oauth2/AzureADAuthenticator.java | 7 +- .../hadoop/tools/mapred/CopyCommitter.java | 8 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 4 - hadoop-tools/hadoop-extras/pom.xml | 10 + hadoop-tools/hadoop-gridmix/pom.xml | 10 + hadoop-tools/hadoop-streaming/pom.xml | 10 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 + .../hadoop-yarn/conf/yarn-env.sh | 1 + .../dev-support/findbugs-exclude.xml | 16 - .../hadoop/yarn/api/ApplicationConstants.java | 9 +- .../yarn/api/records/ResourceUtilization.java | 8 +- .../hadoop-yarn-services-api/pom.xml | 2 +- .../hadoop/yarn/service/webapp/ApiServer.java | 7 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 154 +- .../hadoop-yarn-services-core/pom.xml | 2 +- .../hadoop/yarn/service/ClientAMService.java | 9 +- .../hadoop/yarn/service/ServiceMaster.java | 7 + .../hadoop/yarn/service/ServiceScheduler.java | 67 + .../api/records/ComponentContainers.java | 96 + .../yarn/service/api/records/ConfigFormat.java | 5 +- .../yarn/service/client/ServiceClient.java | 147 +- .../hadoop/yarn/service/utils/ConfigUtils.java | 56 - .../hadoop/yarn/service/utils/FilterUtils.java | 25 +- .../hadoop/yarn/service/utils/HttpUtil.java | 123 ++ .../service/utils/PublishedConfiguration.java | 9 - .../utils/PublishedConfigurationOutputter.java | 2 - .../yarn/service/utils/ServiceApiUtil.java | 8 +- .../hadoop/yarn/service/TestServiceAM.java | 32 +- .../yarn/service/client/TestServiceClient.java | 36 +- .../yarn/service/utils/TestFilterUtils.java | 39 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 10 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 25 +- .../hadoop/yarn/webapp/view/JQueryUI.java | 6 +- .../static/dt-1.10.18/css/custom_datatable.css | 68 + .../webapps/static/dt-1.10.18/css/demo_page.css | 108 ++ .../static/dt-1.10.18/css/demo_table.css | 544 ++++++ .../static/dt-1.10.18/css/jquery.dataTables.css | 466 +++++ .../webapps/static/dt-1.10.18/css/jui-dt.css | 352 ++++ .../static/dt-1.10.18/images/Sorting icons.psd | Bin 0 -> 27490 bytes .../static/dt-1.10.18/images/back_disabled.jpg | Bin 0 -> 612 bytes .../static/dt-1.10.18/images/back_enabled.jpg | Bin 0 -> 807 bytes .../static/dt-1.10.18/images/favicon.ico | Bin 0 -> 894 bytes .../dt-1.10.18/images/forward_disabled.jpg | Bin 0 -> 635 bytes .../dt-1.10.18/images/forward_enabled.jpg | Bin 0 -> 852 bytes .../static/dt-1.10.18/images/sort_asc.png | Bin 0 -> 263 bytes .../dt-1.10.18/images/sort_asc_disabled.png | Bin 0 -> 252 bytes .../static/dt-1.10.18/images/sort_both.png | Bin 0 -> 282 bytes .../static/dt-1.10.18/images/sort_desc.png | Bin 0 -> 260 bytes .../dt-1.10.18/images/sort_desc_disabled.png | Bin 0 -> 251 bytes .../dt-1.10.18/js/jquery.dataTables.min.js | 184 ++ .../webapps/static/dt-1.10.7/css/demo_page.css | 110 -- .../webapps/static/dt-1.10.7/css/demo_table.css | 538 ------ .../webapps/static/dt-1.10.7/css/jui-dt.css | 322 ---- .../static/dt-1.10.7/images/Sorting icons.psd | Bin 27490 -> 0 bytes .../static/dt-1.10.7/images/back_disabled.jpg | Bin 612 -> 0 bytes .../static/dt-1.10.7/images/back_enabled.jpg | Bin 807 -> 0 bytes .../webapps/static/dt-1.10.7/images/favicon.ico | Bin 894 -> 0 bytes .../dt-1.10.7/images/forward_disabled.jpg | Bin 635 -> 0 bytes .../static/dt-1.10.7/images/forward_enabled.jpg | Bin 852 -> 0 bytes .../static/dt-1.10.7/images/sort_asc.png | Bin 263 -> 0 bytes .../dt-1.10.7/images/sort_asc_disabled.png | Bin 252 -> 0 bytes .../static/dt-1.10.7/images/sort_both.png | Bin 282 -> 0 bytes .../static/dt-1.10.7/images/sort_desc.png | Bin 260 -> 0 bytes .../dt-1.10.7/images/sort_desc_disabled.png | Bin 251 -> 0 bytes .../dt-1.10.7/js/jquery.dataTables.min.js | 160 -- .../dev-support/findbugs-exclude.xml | 21 + .../hadoop-yarn/hadoop-yarn-csi/pom.xml | 221 +++ .../hadoop/yarn/csi/client/CsiClient.java | 39 + .../hadoop/yarn/csi/client/CsiClientImpl.java | 51 + .../hadoop/yarn/csi/client/CsiGrpcClient.java | 126 ++ .../hadoop/yarn/csi/client/package-info.java | 21 + .../apache/hadoop/yarn/csi/package-info.java | 21 + .../hadoop/yarn/csi/utils/GrpcHelper.java | 52 + .../hadoop/yarn/csi/utils/package-info.java | 21 + .../hadoop-yarn-csi/src/main/proto/csi.proto | 1114 +++++++++++ .../hadoop/yarn/csi/client/FakeCsiDriver.java | 65 + .../yarn/csi/client/FakeCsiIdentityService.java | 42 + .../hadoop/yarn/csi/client/TestCsiClient.java | 77 + .../hadoop/yarn/csi/client/package-info.java | 21 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 235 +-- .../apache/hadoop/registry/cli/RegistryCli.java | 497 ----- .../hadoop/registry/client/api/BindFlags.java | 41 - .../registry/client/api/DNSOperations.java | 60 - .../client/api/DNSOperationsFactory.java | 78 - .../registry/client/api/RegistryConstants.java | 390 ---- .../registry/client/api/RegistryOperations.java | 182 -- .../client/api/RegistryOperationsFactory.java | 160 -- .../registry/client/api/package-info.java | 35 - .../registry/client/binding/JsonSerDeser.java | 117 -- .../client/binding/RegistryPathUtils.java | 238 --- .../client/binding/RegistryTypeUtils.java | 291 --- .../registry/client/binding/RegistryUtils.java | 399 ---- .../registry/client/binding/package-info.java | 22 - .../AuthenticationFailedException.java | 39 - .../exceptions/InvalidPathnameException.java | 40 - .../exceptions/InvalidRecordException.java | 41 - .../NoChildrenForEphemeralsException.java | 48 - .../exceptions/NoPathPermissionsException.java | 45 - .../client/exceptions/NoRecordException.java | 45 - .../client/exceptions/RegistryIOException.java | 58 - .../client/exceptions/package-info.java | 33 - .../impl/FSRegistryOperationsService.java | 249 --- .../client/impl/RegistryOperationsClient.java | 55 - .../registry/client/impl/package-info.java | 26 - .../client/impl/zk/BindingInformation.java | 41 - .../registry/client/impl/zk/CuratorService.java | 896 --------- .../registry/client/impl/zk/ListenerHandle.java | 25 - .../registry/client/impl/zk/PathListener.java | 30 - .../client/impl/zk/RegistryBindingSource.java | 36 - .../impl/zk/RegistryInternalConstants.java | 81 - .../impl/zk/RegistryOperationsService.java | 165 -- .../client/impl/zk/RegistrySecurity.java | 1143 ----------- .../registry/client/impl/zk/ZKPathDumper.java | 133 -- .../client/impl/zk/ZookeeperConfigOptions.java | 118 -- .../registry/client/impl/zk/package-info.java | 39 - .../registry/client/types/AddressTypes.java | 94 - .../hadoop/registry/client/types/Endpoint.java | 247 --- .../registry/client/types/ProtocolTypes.java | 99 - .../client/types/RegistryPathStatus.java | 123 -- .../registry/client/types/ServiceRecord.java | 312 --- .../registry/client/types/package-info.java | 41 - .../client/types/yarn/PersistencePolicies.java | 50 - .../types/yarn/YarnRegistryAttributes.java | 42 - .../dns/ApplicationServiceRecordProcessor.java | 366 ---- .../server/dns/BaseServiceRecordProcessor.java | 470 ----- .../dns/ContainerServiceRecordProcessor.java | 284 --- .../hadoop/registry/server/dns/LookupTask.java | 39 - .../dns/PrivilegedRegistryDNSStarter.java | 80 - .../server/dns/RecordCreatorFactory.java | 275 --- .../hadoop/registry/server/dns/RegistryDNS.java | 1800 ------------------ .../registry/server/dns/RegistryDNSServer.java | 267 --- .../registry/server/dns/ReverseZoneUtils.java | 171 -- .../registry/server/dns/SecureableZone.java | 151 -- .../server/dns/ServiceRecordProcessor.java | 53 - .../registry/server/dns/ZoneSelector.java | 33 - .../registry/server/dns/package-info.java | 26 - .../integration/SelectByYarnPersistence.java | 60 - .../server/integration/package-info.java | 23 - .../hadoop/registry/server/package-info.java | 27 - .../server/services/AddingCompositeService.java | 56 - .../services/DeleteCompletionCallback.java | 57 - .../server/services/MicroZookeeperService.java | 282 --- .../services/MicroZookeeperServiceKeys.java | 69 - .../server/services/RegistryAdminService.java | 529 ----- .../registry/server/services/package-info.java | 39 - .../src/main/resources/.keep | 0 .../src/main/tla/yarnregistry.tla | 582 ------ .../hadoop/registry/AbstractRegistryTest.java | 118 -- .../hadoop/registry/AbstractZKRegistryTest.java | 113 -- .../hadoop/registry/RegistryTestHelper.java | 353 ---- .../hadoop/registry/cli/TestRegistryCli.java | 197 -- .../client/binding/TestMarshalling.java | 143 -- .../binding/TestRegistryOperationUtils.java | 56 - .../client/binding/TestRegistryPathUtils.java | 186 -- .../client/impl/CuratorEventCatcher.java | 68 - .../client/impl/TestCuratorService.java | 249 --- .../impl/TestFSRegistryOperationsService.java | 298 --- .../client/impl/TestMicroZookeeperService.java | 60 - .../integration/TestYarnPolicySelector.java | 65 - .../operations/TestRegistryOperations.java | 331 ---- .../secure/AbstractSecureRegistryTest.java | 368 ---- .../registry/secure/KerberosConfiguration.java | 100 - .../secure/TestRegistrySecurityHelper.java | 211 -- .../registry/secure/TestSecureLogins.java | 231 --- .../registry/secure/TestSecureRegistry.java | 213 --- .../registry/server/dns/TestRegistryDNS.java | 725 ------- .../server/dns/TestReverseZoneUtils.java | 89 - .../server/dns/TestSecureRegistryDNS.java | 44 - .../test/resources/0.17.172.in-addr.arpa.zone | 36 - .../src/test/resources/log4j.properties | 63 - .../src/test/resources/test.private | 32 - .../hadoop-yarn-server-common/pom.xml | 2 +- .../hadoop/yarn/server/AMRMClientRelayer.java | 55 +- .../policies/router/LoadBasedRouterPolicy.java | 6 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 28 + .../server/uam/UnmanagedApplicationManager.java | 28 +- .../yarn/server/MockResourceManagerFacade.java | 5 +- .../yarn/server/TestAMRMClientRelayer.java | 10 +- .../router/TestLoadBasedRouterPolicy.java | 31 + .../metrics/TestAMRMClientRelayerMetrics.java | 6 - .../uam/TestUnmanagedApplicationManager.java | 27 +- .../hadoop-yarn-server-nodemanager/pom.xml | 2 +- .../server/nodemanager/ContainerExecutor.java | 12 + .../nodemanager/DefaultContainerExecutor.java | 7 + .../nodemanager/LinuxContainerExecutor.java | 43 + .../amrmproxy/FederationInterceptor.java | 18 +- .../linux/privileged/PrivilegedOperation.java | 6 +- .../linux/resources/CGroupsHandler.java | 2 +- .../linux/resources/CGroupsHandlerImpl.java | 4 +- .../CGroupsMemoryResourceHandlerImpl.java | 25 - .../linux/resources/DefaultOOMHandler.java | 45 +- .../linux/resources/MemoryResourceHandler.java | 10 - .../runtime/DockerLinuxContainerRuntime.java | 18 + .../linux/runtime/docker/DockerRunCommand.java | 6 + .../monitor/ContainersMonitor.java | 6 +- .../monitor/ContainersMonitorImpl.java | 119 +- ...locationBasedResourceUtilizationTracker.java | 27 +- .../scheduler/ContainerScheduler.java | 5 +- .../nodemanager/webapp/NMWebServices.java | 27 + .../impl/container-executor.c | 127 +- .../impl/container-executor.h | 22 +- .../main/native/container-executor/impl/main.c | 30 +- .../impl/utils/string-utils.c | 9 + .../impl/utils/string-utils.h | 6 + .../test/test-container-executor.c | 115 +- .../nodemanager/TestLinuxContainerExecutor.java | 11 + .../amrmproxy/TestAMRMProxyService.java | 1 - .../TestContainerManagerRecovery.java | 2 +- .../linux/resources/TestCGroupsHandlerImpl.java | 2 +- .../TestCGroupsMemoryResourceHandlerImpl.java | 44 - .../linux/resources/TestDefaultOOMHandler.java | 434 ++++- .../TestContainersMonitorResourceChange.java | 4 + ...locationBasedResourceUtilizationTracker.java | 18 - .../TestContainerSchedulerRecovery.java | 79 +- .../server/resourcemanager/ClientRMService.java | 12 +- .../server/resourcemanager/RMAuditLogger.java | 81 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 19 +- .../scheduler/fair/FSAppAttempt.java | 5 +- .../scheduler/fifo/FifoAppAttempt.java | 10 +- .../webapp/dao/ReservationListInfo.java | 4 + .../resourcemanager/TestRMAuditLogger.java | 23 +- .../capacity/TestContainerAllocation.java | 4 + .../security/TestRMDelegationTokens.java | 12 +- .../hadoop/yarn/server/router/Router.java | 8 + .../hadoop/yarn/server/router/TestRouter.java | 38 + .../pom.xml | 8 + .../pom.xml | 8 + .../collector/TimelineCollectorManager.java | 2 +- .../src/site/markdown/CapacityScheduler.md | 2 +- .../src/site/markdown/DockerContainers.md | 16 + .../src/site/markdown/NodeManager.md | 10 +- .../site/markdown/NodeManagerCGroupsMemory.md | 12 +- .../src/site/markdown/NodeManagerCgroups.md | 4 +- .../src/site/markdown/TimelineServiceV2.md | 17 + .../src/site/markdown/registry/index.md | 30 - .../markdown/registry/registry-configuration.md | 397 ---- .../site/markdown/registry/registry-security.md | 120 -- .../registry/using-the-yarn-service-registry.md | 273 --- .../src/site/markdown/registry/yarn-registry.md | 1018 ---------- .../src/site/markdown/yarn-service/Overview.md | 2 +- .../site/markdown/yarn-service/RegistryDNS.md | 200 -- .../markdown/yarn-service/ServiceDiscovery.md | 83 +- .../markdown/yarn-service/ServiceUpgrade.md | 38 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 1 + pom.xml | 2 - 758 files changed, 33956 insertions(+), 25918 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 9fd5c63,abacafe..b41baa9 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@@ -97,38 -97,11 +97,43 @@@ public final class HddsConfigKeys "hdds.lock.max.concurrency"; public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100; + public static final String HDDS_KEY_LEN = "hdds.key.len"; + public static final int HDDS_DEFAULT_KEY_LEN = 2048; + public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo"; + public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA"; + public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider"; + public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC"; + public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name"; + public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys"; + + // TODO : Talk to StorageIO classes and see if they can return a secure + // storage location for each node. + public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir"; + public static final String HDDS_PRIVATE_KEY_FILE_NAME = + "hdds.priv.key.file.name"; + public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem"; + public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file" + + ".name"; + public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem"; + + /** + * Maximum duration of certificates issued by SCM including Self-Signed Roots. + * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS + * Default value is 5 years and written as P1865D. + */ + public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration"; + // Limit Certificate duration to a max value of 5 years. + public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D"; + + public static final String HDDS_X509_SIGNATURE_ALGO = + "hdds.x509.signature.algorithm"; + public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA"; + - - + // This configuration setting is used as a fallback location by all + // Ozone/HDDS services for their metadata. It is useful as a single + // config point for test/PoC clusters. + // + // In any real cluster where performance matters, the SCM, OM and DN + // metadata locations must be configured explicitly. + public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java index 9231c8a,0000000..9e0231a mode 100644,000000..100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java @@@ -1,231 -1,0 +1,231 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.security.x509; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.Provider; +import java.security.Security; +import java.time.Duration; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT; - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; ++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; + +/** + * A class that deals with all Security related configs in HDDS. + * + * This class allows security configs to be read and used consistently across + * all of security related code base. + */ +public class SecurityConfig { + private static final Logger LOG = + LoggerFactory.getLogger(SecurityConfig.class); + private static volatile Provider provider; + private final Configuration configuration; + private final int size; + private final String keyAlgo; + private final String providerString; + private final String metadatDir; + private final String keyDir; + private final String privateKeyFileName; + private final String publicKeyFileName; + private final Duration certDuration; + private final String x509SignatureAlgo; + + /** + * Constructs a SecurityConfig. + * + * @param configuration - HDDS Configuration + */ + public SecurityConfig(Configuration configuration) { + Preconditions.checkNotNull(configuration, "Configuration cannot be null"); + this.configuration = configuration; + this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN); + this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM, + HDDS_DEFAULT_KEY_ALGORITHM); + this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER, + HDDS_DEFAULT_SECURITY_PROVIDER); + + // Please Note: To make it easy for our customers we will attempt to read + // HDDS metadata dir and if that is not set, we will use Ozone directory. + // TODO: We might want to fix this later. + this.metadatDir = this.configuration.get(HDDS_METADATA_DIR_NAME, + configuration.get(OZONE_METADATA_DIRS)); + + Preconditions.checkNotNull(this.metadatDir, "Metadata directory can't be" + + " null. Please check configs."); + this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME, + HDDS_KEY_DIR_NAME_DEFAULT); + this.privateKeyFileName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME, + HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT); + this.publicKeyFileName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME, + HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT); + + String durationString = this.configuration.get(HDDS_X509_MAX_DURATION, + HDDS_X509_MAX_DURATION_DEFAULT); + this.certDuration = Duration.parse(durationString); + this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO, + HDDS_X509_SIGNATURE_ALGO_DEFAULT); + + // First Startup -- if the provider is null, check for the provider. + if (SecurityConfig.provider == null) { + synchronized (SecurityConfig.class) { + provider = Security.getProvider(this.providerString); + if (SecurityConfig.provider == null) { + // Provider not found, let us try to Dynamically initialize the + // provider. + provider = initSecurityProvider(this.providerString); + } + } + } + } + + /** + * Returns the public key file name, This is used for storing the public + * keys on disk. + * + * @return String, File name used for public keys. + */ + public String getPublicKeyFileName() { + return publicKeyFileName; + } + + /** + * Returns the private key file name.This is used for storing the private + * keys on disk. + * + * @return String, File name used for private keys. + */ + public String getPrivateKeyFileName() { + return privateKeyFileName; + } + + /** + * Returns the File path to where keys are stored. + * + * @return String Key location. + */ + public Path getKeyLocation() { + return Paths.get(metadatDir, keyDir); + } + + /** + * Gets the Key Size, The default key size is 2048, since the default + * algorithm used is RSA. User can change this by setting the "hdds.key + * .len" in configuration. + * + * @return key size. + */ + public int getSize() { + return size; + } + + /** + * Returns the Provider name. SCM defaults to using Bouncy Castle and will + * return "BC". + * + * @return String Provider name. + */ + public String getProvider() { + return providerString; + } + + /** + * Returns the Key generation Algorithm used. User can change this by + * setting the "hdds.key.algo" in configuration. + * + * @return String Algo. + */ + public String getKeyAlgo() { + return keyAlgo; + } + + /** + * Returns the X.509 Signature Algorithm used. This can be changed by setting + * "hdds.x509.signature.algorithm" to the new name. The default algorithm + * is SHA256withRSA. + * + * @return String + */ + public String getSignatureAlgo() { + return x509SignatureAlgo; + } + + /** + * Returns the Configuration used for initializing this SecurityConfig. + * + * @return Configuration + */ + public Configuration getConfiguration() { + return configuration; + } + + /** + * Returns the maximum length a certificate can be valid in SCM. The + * default value is 5 years. This can be changed by setting + * "hdds.x509.max.duration" in configuration. The formats accepted are + * based on the ISO-8601 duration format PnDTnHnMn.nS + * + * Default value is 5 years and written as P1865D. + * + * @return Duration. + */ + public Duration getMaxCertificateDuration() { + return this.certDuration; + } + + /** + * Adds a security provider dynamically if it is not loaded already. + * + * @param providerName - name of the provider. + */ + private Provider initSecurityProvider(String providerName) { + switch (providerName) { + case "BC": + Security.addProvider(new BouncyCastleProvider()); + return Security.getProvider(providerName); + default: + LOG.error("Security Provider:{} is unknown", provider); + throw new SecurityException("Unknown security provider:" + provider); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java index 0b9ef31,0000000..25f84ff mode 100644,000000..100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java @@@ -1,285 -1,0 +1,285 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.security.x509.certificates; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException; +import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; +import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil; +import org.bouncycastle.asn1.ASN1Sequence; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.Extensions; +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; +import org.bouncycastle.operator.ContentVerifierProvider; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.bouncycastle.pkcs.PKCSException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.util.UUID; + - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; ++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; + +public class TestCertificateSignRequest { + + private SecurityConfig securityConfig; + private static OzoneConfiguration conf = new OzoneConfiguration(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void init() throws IOException { + conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString()); + securityConfig = new SecurityConfig(conf); + } + + @Test + public void testGenerateCSR() throws NoSuchProviderException, + NoSuchAlgorithmException, SCMSecurityException, + OperatorCreationException, PKCSException { + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "DN001"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + CertificateSignRequest.Builder builder = + new CertificateSignRequest.Builder() + .setSubject(subject) + .setScmID(scmID) + .setClusterID(clusterID) + .setKey(keyPair) + .setConfiguration(conf); + PKCS10CertificationRequest csr = builder.build(); + + // Check the Subject Name is in the expected format. + String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(), + subject, scmID, clusterID); + Assert.assertEquals(csr.getSubject().toString(), dnName); + + // Verify the public key info match + byte[] encoded = keyPair.getPublic().getEncoded(); + SubjectPublicKeyInfo subjectPublicKeyInfo = + SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded)); + SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo(); + Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo); + + // Verify CSR with attribute for extensions + Assert.assertEquals(1, csr.getAttributes().length); + Extensions extensions = SecurityUtil.getPkcs9Extensions(csr); + + // Verify basic constraints extension + Extension basicExt = extensions.getExtension(Extension + .basicConstraints); + Assert.assertEquals(true, basicExt.isCritical()); + + // Verify key usage extension + Extension keyUsageExt = extensions.getExtension(Extension.keyUsage); + Assert.assertEquals(true, keyUsageExt.isCritical()); + + + // Verify San extension not set + Assert.assertEquals(null, + extensions.getExtension(Extension.subjectAlternativeName)); + + // Verify signature in CSR + ContentVerifierProvider verifierProvider = + new JcaContentVerifierProviderBuilder().setProvider(securityConfig + .getProvider()).build(csr.getSubjectPublicKeyInfo()); + Assert.assertEquals(true, csr.isSignatureValid(verifierProvider)); + } + + @Test + public void testGenerateCSRwithSan() throws NoSuchProviderException, + NoSuchAlgorithmException, SCMSecurityException, + OperatorCreationException, PKCSException { + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "DN001"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + CertificateSignRequest.Builder builder = + new CertificateSignRequest.Builder() + .setSubject(subject) + .setScmID(scmID) + .setClusterID(clusterID) + .setKey(keyPair) + .setConfiguration(conf); + + // Multi-home + builder.addIpAddress("192.168.1.1"); + builder.addIpAddress("192.168.2.1"); + + builder.addDnsName("dn1.abc.com"); + builder.addRfc822Name("t...@abc.com"); + + PKCS10CertificationRequest csr = builder.build(); + + // Check the Subject Name is in the expected format. + String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(), + subject, scmID, clusterID); + Assert.assertEquals(csr.getSubject().toString(), dnName); + + // Verify the public key info match + byte[] encoded = keyPair.getPublic().getEncoded(); + SubjectPublicKeyInfo subjectPublicKeyInfo = + SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded)); + SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo(); + Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo); + + // Verify CSR with attribute for extensions + Assert.assertEquals(1, csr.getAttributes().length); + Extensions extensions = SecurityUtil.getPkcs9Extensions(csr); + + // Verify key usage extension + Extension sanExt = extensions.getExtension(Extension.keyUsage); + Assert.assertEquals(true, sanExt.isCritical()); + + + // Verify signature in CSR + ContentVerifierProvider verifierProvider = + new JcaContentVerifierProviderBuilder().setProvider(securityConfig + .getProvider()).build(csr.getSubjectPublicKeyInfo()); + Assert.assertEquals(true, csr.isSignatureValid(verifierProvider)); + } + + @Test + public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException, + NoSuchAlgorithmException, SCMSecurityException { + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "DN001"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + CertificateSignRequest.Builder builder = + new CertificateSignRequest.Builder() + .setSubject(subject) + .setScmID(scmID) + .setClusterID(clusterID) + .setKey(keyPair) + .setConfiguration(conf); + + try { + builder.setKey(null); + builder.build(); + Assert.fail("Null Key should have failed."); + } catch (NullPointerException | IllegalArgumentException e) { + builder.setKey(keyPair); + } + + // Now try with blank/null Subject. + try { + builder.setSubject(null); + builder.build(); + Assert.fail("Null/Blank Subject should have thrown."); + } catch (IllegalArgumentException e) { + builder.setSubject(subject); + } + + try { + builder.setSubject(""); + builder.build(); + Assert.fail("Null/Blank Subject should have thrown."); + } catch (IllegalArgumentException e) { + builder.setSubject(subject); + } + + // Now try with blank/null SCM ID + try { + builder.setScmID(null); + builder.build(); + Assert.fail("Null/Blank SCM ID should have thrown."); + } catch (IllegalArgumentException e) { + builder.setScmID(scmID); + } + + // Now try with blank/null SCM ID + try { + builder.setClusterID(null); + builder.build(); + Assert.fail("Null/Blank Cluster ID should have thrown."); + } catch (IllegalArgumentException e) { + builder.setClusterID(clusterID); + } + + // Now try with invalid IP address + try { + builder.addIpAddress("255.255.255.*"); + builder.build(); + Assert.fail("Invalid ip address"); + } catch (IllegalArgumentException e) { + } + + PKCS10CertificationRequest csr = builder.build(); + + // Check the Subject Name is in the expected format. + String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(), + subject, scmID, clusterID); + Assert.assertEquals(csr.getSubject().toString(), dnName); + + // Verify the public key info match + byte[] encoded = keyPair.getPublic().getEncoded(); + SubjectPublicKeyInfo subjectPublicKeyInfo = + SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded)); + SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo(); + Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo); + + // Verify CSR with attribute for extensions + Assert.assertEquals(1, csr.getAttributes().length); + } + + @Test + public void testCsrSerialization() throws NoSuchProviderException, + NoSuchAlgorithmException, SCMSecurityException, IOException { + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "DN001"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + CertificateSignRequest.Builder builder = + new CertificateSignRequest.Builder() + .setSubject(subject) + .setScmID(scmID) + .setClusterID(clusterID) + .setKey(keyPair) + .setConfiguration(conf); + PKCS10CertificationRequest csr = builder.build(); + byte[] csrBytes = csr.getEncoded(); + + // Verify de-serialized CSR matches with the original CSR + PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes); + Assert.assertEquals(csr, dsCsr); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java index 3a00ca0,0000000..5f99e2c mode 100644,000000..100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java @@@ -1,258 -1,0 +1,258 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.security.x509.certificates; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException; +import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.math.BigInteger; +import java.security.InvalidKeyException; +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SignatureException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.UUID; + - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; ++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; + +/** + * Test Class for Root Certificate generation. + */ +public class TestRootCertificate { + private SecurityConfig securityConfig; + private static OzoneConfiguration conf = new OzoneConfiguration(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void init() throws IOException { + conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString()); + securityConfig = new SecurityConfig(conf); + } + + @Test + public void testAllFieldsAreExpected() + throws SCMSecurityException, NoSuchProviderException, + NoSuchAlgorithmException, CertificateException, + SignatureException, InvalidKeyException { + Instant now = Instant.now(); + Date notBefore = Date.from(now); + Date notAfter = Date.from(now.plus(Duration.ofDays(365))); + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "testRootCert"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + SelfSignedCertificate.Builder builder = + SelfSignedCertificate.newBuilder() + .setBeginDate(notBefore) + .setEndDate(notAfter) + .setClusterID(clusterID) + .setScmID(scmID) + .setSubject(subject) + .setKey(keyPair) + .setConfiguration(conf); + + X509CertificateHolder certificateHolder = builder.build(); + + //Assert that we indeed have a self signed certificate. + Assert.assertEquals(certificateHolder.getIssuer(), + certificateHolder.getSubject()); + + + // Make sure that NotBefore is before the current Date + Date invalidDate = Date.from(now.minus(Duration.ofDays(1))); + Assert.assertFalse( + certificateHolder.getNotBefore() + .before(invalidDate)); + + //Make sure the end date is honored. + invalidDate = Date.from(now.plus(Duration.ofDays(366))); + Assert.assertFalse( + certificateHolder.getNotAfter() + .after(invalidDate)); + + // Check the Subject Name and Issuer Name is in the expected format. + String dnName = String.format(SelfSignedCertificate.getNameFormat(), + subject, scmID, clusterID); + Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName); + Assert.assertEquals(certificateHolder.getSubject().toString(), dnName); + + // We did not ask for this Certificate to be a CA certificate, hence that + // extension should be null. + Assert.assertNull( + certificateHolder.getExtension(Extension.basicConstraints)); + + // Extract the Certificate and verify that certificate matches the public + // key. + X509Certificate cert = + new JcaX509CertificateConverter().getCertificate(certificateHolder); + cert.verify(keyPair.getPublic()); + } + + @Test + public void testCACert() + throws SCMSecurityException, NoSuchProviderException, + NoSuchAlgorithmException { + Instant now = Instant.now(); + Date notBefore = Date.from(now); + Date notAfter = Date.from(now.plus(Duration.ofDays(365))); + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "testRootCert"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + SelfSignedCertificate.Builder builder = + SelfSignedCertificate.newBuilder() + .setBeginDate(notBefore) + .setEndDate(notAfter) + .setClusterID(clusterID) + .setScmID(scmID) + .setSubject(subject) + .setKey(keyPair) + .setConfiguration(conf) + .makeCA(); + + X509CertificateHolder certificateHolder = builder.build(); + // This time we asked for a CA Certificate, make sure that extension is + // present and valid. + Extension basicExt = + certificateHolder.getExtension(Extension.basicConstraints); + + Assert.assertNotNull(basicExt); + Assert.assertTrue(basicExt.isCritical()); + + // Since this code assigns ONE for the root certificate, we check if the + // serial number is the expected number. + Assert.assertEquals(certificateHolder.getSerialNumber(), BigInteger.ONE); + } + + @Test + public void testInvalidParamFails() + throws SCMSecurityException, NoSuchProviderException, + NoSuchAlgorithmException { + Instant now = Instant.now(); + Date notBefore = Date.from(now); + Date notAfter = Date.from(now.plus(Duration.ofDays(365))); + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "testRootCert"; + HDDSKeyGenerator keyGen = + new HDDSKeyGenerator(securityConfig.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + + SelfSignedCertificate.Builder builder = + SelfSignedCertificate.newBuilder() + .setBeginDate(notBefore) + .setEndDate(notAfter) + .setClusterID(clusterID) + .setScmID(scmID) + .setSubject(subject) + .setConfiguration(conf) + .setKey(keyPair) + .makeCA(); + try { + builder.setKey(null); + builder.build(); + Assert.fail("Null Key should have failed."); + } catch (NullPointerException | IllegalArgumentException e) { + builder.setKey(keyPair); + } + + // Now try with Blank Subject. + try { + builder.setSubject(""); + builder.build(); + Assert.fail("Null/Blank Subject should have thrown."); + } catch (IllegalArgumentException e) { + builder.setSubject(subject); + } + + // Now try with blank/null SCM ID + try { + builder.setScmID(null); + builder.build(); + Assert.fail("Null/Blank SCM ID should have thrown."); + } catch (IllegalArgumentException e) { + builder.setScmID(scmID); + } + + + // Now try with blank/null SCM ID + try { + builder.setClusterID(null); + builder.build(); + Assert.fail("Null/Blank Cluster ID should have thrown."); + } catch (IllegalArgumentException e) { + builder.setClusterID(clusterID); + } + + + // Swap the Begin and End Date and verify that we cannot create a + // certificate like that. + try { + builder.setBeginDate(notAfter); + builder.setEndDate(notBefore); + builder.build(); + Assert.fail("Illegal dates should have thrown."); + } catch (IllegalArgumentException e) { + builder.setBeginDate(notBefore); + builder.setEndDate(notAfter); + } + + try { + KeyPair newKey = keyGen.generateKey(); + KeyPair wrongKey = new KeyPair(newKey.getPublic(), keyPair.getPrivate()); + builder.setKey(wrongKey); + X509CertificateHolder certificateHolder = builder.build(); + X509Certificate cert = + new JcaX509CertificateConverter().getCertificate(certificateHolder); + cert.verify(wrongKey.getPublic()); + Assert.fail("Invalid Key, should have thrown."); + } catch (SCMSecurityException | CertificateException + | SignatureException | InvalidKeyException e) { + builder.setKey(keyPair); + } + // Assert that we can create a certificate with all sane params. + Assert.assertNotNull(builder.build()); + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java index f9541a2,0000000..08761f4 mode 100644,000000..100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java @@@ -1,87 -1,0 +1,87 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.security.x509.keys; + - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; ++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PublicKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Test class for HDDS Key Generator. + */ +public class TestHDDSKeyGenerator { + private SecurityConfig config; + + @Before + public void init() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath")); + config = new SecurityConfig(conf); + } + /** + * In this test we verify that we are able to create a key pair, then get + * bytes of that and use ASN1. parser to parse it back to a private key. + * @throws NoSuchProviderException - On Error, due to missing Java + * dependencies. + * @throws NoSuchAlgorithmException - On Error, due to missing Java + * dependencies. + */ + @Test + public void testGenerateKey() + throws NoSuchProviderException, NoSuchAlgorithmException { + HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(); + Assert.assertEquals(config.getKeyAlgo(), + keyPair.getPrivate().getAlgorithm()); + PKCS8EncodedKeySpec keySpec = + new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded()); + Assert.assertEquals("PKCS#8", keySpec.getFormat()); + } + + /** + * In this test we assert that size that we specified is used for Key + * generation. + * @throws NoSuchProviderException - On Error, due to missing Java + * dependencies. + * @throws NoSuchAlgorithmException - On Error, due to missing Java + * dependencies. + */ + @Test + public void testGenerateKeyWithSize() throws NoSuchProviderException, + NoSuchAlgorithmException { + HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration()); + KeyPair keyPair = keyGen.generateKey(4096); + PublicKey publicKey = keyPair.getPublic(); + if(publicKey instanceof RSAPublicKey) { + Assert.assertEquals(4096, + ((RSAPublicKey)(publicKey)).getModulus().bitLength()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java index 072ca49,40b3d7a..2cffa74 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java @@@ -145,28 -161,6 +161,28 @@@ public final class HddsServerUtil } /** + * Retrieve the socket address that should be used by scm security server to + * service clients. + * + * @param conf + * @return Target InetSocketAddress for the SCM security service. + */ + public static InetSocketAddress getScmSecurityInetAddress( + Configuration conf) { + final Optional<String> host = getHostNameFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY); + + final Optional<Integer> port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY); + + return NetUtils.createSocketAddr( - host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) + ++ host.orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) + + ":" + port - .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY, ++ .orElse(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY, + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT))); + } + + /** * Retrieve the socket address that should be used by DataNodes to connect * to the SCM. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index 84131fe,aa940df..e8e2b1a --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@@ -32,10 -35,8 +35,9 @@@ import org.apache.hadoop.hdds.scm.serve import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; - import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.common.Storage.StorageState; import org.apache.hadoop.ozone.container.common.SCMTestUtils; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org