This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 9c4008d5404d310ddc4e7cf5be22b80156692ebf Merge: 132911c 0b4779c Author: Elek Márton <[email protected]> AuthorDate: Tue May 25 13:30:16 2021 +0200 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec .github/buildenv/Dockerfile | 20 -- .github/buildenv/entrypoint.sh | 18 - .github/workflows/post-commit.yml | 117 +++++-- .gitignore | 3 + CONTRIBUTING.md | 10 +- NOTICE.txt | 2 +- README.md | 2 +- .../hadoop/hdds/scm/XceiverClientFactory.java | 3 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +- hadoop-hdds/common/pom.xml | 5 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 95 +++-- .../hadoop/hdds/client/RatisReplicationConfig.java | 15 + .../hadoop/hdds/client/ReplicationConfig.java | 55 +++ .../hdds/client/StandaloneReplicationConfig.java | 9 + .../hdds/ratis/ServerNotLeaderException.java | 15 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +- .../hdds/scm/container/ContainerException.java | 14 +- .../scm/container/ContainerNotFoundException.java | 4 +- .../ContainerReplicaNotFoundException.java | 4 +- .../hadoop/hdds/scm/exceptions/SCMException.java | 7 +- .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 58 ++- .../scm/pipeline/PipelineNotFoundException.java | 8 +- .../pipeline/UnknownPipelineStateException.java | 8 +- .../protocol/StorageContainerLocationProtocol.java | 18 + .../security/token/ContainerTokenIdentifier.java | 103 ++++++ .../security/token/OzoneBlockTokenIdentifier.java | 146 +++----- .../security/token/ShortLivedTokenIdentifier.java | 112 ++++++ .../hadoop/hdds/security/x509/SecurityConfig.java | 16 + .../hadoop/hdds/security/x509/crl}/CRLCodec.java | 2 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 1 - .../hdds/security/x509/crl}/CRLInfoCodec.java | 3 +- .../hdds/security/x509/crl/package-info.java | 24 ++ .../org/apache/hadoop/hdds/utils/db/Codec.java | 0 .../apache/hadoop/hdds/utils/db/package-info.java | 22 ++ .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 10 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 + .../common/src/main/resources/ozone-default.xml | 164 ++++++--- .../hadoop/hdds/client/TestReplicationConfig.java | 147 ++++++++ .../hdds/ratis/TestServerNotLeaderException.java | 82 +++++ .../ozone/container/ContainerTestHelper.java | 146 ++++---- hadoop-hdds/container-service/pom.xml | 12 + .../hdds/datanode/metadata/CRLDBDefinition.java | 99 ++++++ .../hdds/datanode/metadata/DatanodeCRLStore.java | 84 +++++ .../datanode/metadata/DatanodeCRLStoreImpl.java | 127 +++++++ .../hdds/datanode/metadata/package-info.java | 22 ++ .../apache/hadoop/ozone/HddsDatanodeService.java | 19 +- .../container/common/impl/HddsDispatcher.java | 25 +- .../common/statemachine/DatanodeConfiguration.java | 30 ++ .../states/endpoint/HeartbeatEndpointTask.java | 2 +- .../states/endpoint/RegisterEndpointTask.java | 2 +- .../container/common/utils/HddsVolumeUtil.java | 11 + .../ozone/container/common/volume/HddsVolume.java | 13 + .../container/common/volume/MutableVolumeSet.java | 28 +- .../container/keyvalue/KeyValueContainer.java | 6 + .../container/keyvalue/KeyValueContainerData.java | 2 + .../ozone/container/keyvalue/KeyValueHandler.java | 3 +- .../container/keyvalue/helpers/BlockUtils.java | 4 +- .../container/keyvalue/helpers/ChunkUtils.java | 42 ++- .../keyvalue/impl/ChunkManagerDispatcher.java | 10 +- .../keyvalue/impl/ChunkManagerFactory.java | 5 +- .../keyvalue/impl/FilePerBlockStrategy.java | 32 +- .../keyvalue/impl/FilePerChunkStrategy.java | 15 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 21 +- .../stream/DirectoryServerDestination.java | 35 +- .../container/stream/DirectoryServerSource.java | 57 +++ .../container/stream/DirstreamClientHandler.java | 143 ++++++++ .../container/stream/DirstreamServerHandler.java | 135 +++++++ .../ozone/container/stream/StreamingClient.java | 101 ++++++ .../container/stream/StreamingDestination.java | 31 +- .../ozone/container/stream/StreamingServer.java | 94 +++++ .../ozone/container/stream/StreamingSource.java | 35 +- .../ozone/container/stream/package-info.java | 22 ++ .../metadata/TestDatanodeCRLStoreImpl.java | 127 +++++++ .../hdds/datanode/metadata/package-info.java} | 31 +- .../hadoop/ozone/TestHddsDatanodeService.java | 6 +- .../container/common/TestBlockDeletingService.java | 47 ++- .../TestSchemaOneBackwardsCompatibility.java | 18 + .../impl/TestContainerDeletionChoosingPolicy.java | 31 +- .../common/impl/TestContainerPersistence.java | 3 +- .../statemachine/TestDatanodeConfiguration.java | 15 + .../container/keyvalue/ChunkLayoutTestInfo.java | 4 +- .../container/keyvalue/helpers/TestChunkUtils.java | 14 +- .../keyvalue/impl/TestFilePerChunkStrategy.java | 3 +- .../container/stream/TestStreamingServer.java | 127 +++++++ hadoop-hdds/docs/content/interface/ReconApi.md | 390 +++++++++------------ hadoop-hdds/docs/content/interface/ReconApi.zh.md | 356 ++++++++----------- hadoop-hdds/docs/content/interface/S3.md | 21 ++ hadoop-hdds/framework/pom.xml | 6 + .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 11 + .../SCMSecurityProtocolClientSideTranslatorPB.java | 14 + .../hadoop/hdds/scm/protocolPB/OzonePBHelper.java | 82 +++++ ...inerLocationProtocolClientSideTranslatorPB.java | 64 +++- .../SCMBlockLocationFailoverProxyProvider.java | 136 +++---- .../SCMContainerLocationFailoverProxyProvider.java | 158 +++++---- .../SCMSecurityProtocolFailoverProxyProvider.java | 101 +++--- .../hadoop/hdds}/security/OzoneSecretKey.java | 4 +- .../hadoop/hdds}/security/OzoneSecretManager.java | 2 +- .../hdds}/security/OzoneSecurityException.java | 2 +- .../apache/hadoop/hdds/security/package-info.java | 31 +- .../hdds/security/token/BlockTokenVerifier.java | 168 +++------ .../security/token/CompositeTokenVerifier.java | 47 +++ .../token/ContainerTokenSecretManager.java | 48 +++ .../security/token/ContainerTokenVerifier.java | 51 +++ .../hdds/security/token/NoopTokenVerifier.java | 38 +- .../token/ShortLivedTokenSecretManager.java | 107 ++++++ .../security/token/ShortLivedTokenVerifier.java | 135 +++++++ .../hadoop/hdds/security/token/TokenVerifier.java | 64 +++- .../certificate/authority/CertificateServer.java | 4 +- .../certificate/authority/DefaultCAServer.java | 3 +- .../certificate/authority/DefaultCRLApprover.java | 2 +- .../x509/certificate/client/CertificateClient.java | 15 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 2 + .../hadoop/hdds/utils/DBCheckpointServlet.java | 30 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 155 ++++---- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 4 +- .../hdds/scm/protocol/TestReplicationConfig.java | 86 ----- .../security/token/TestBlockTokenVerifier.java | 75 ++++ .../security/token/TestContainerTokenVerifier.java | 75 ++++ .../token/TestOzoneBlockTokenIdentifier.java | 2 +- .../hdds/security/token/TokenVerifierTests.java | 242 +++++++++++++ .../certificate/authority/TestDefaultCAServer.java | 6 +- .../x509/certificate/utils/TestCRLCodec.java | 1 + .../src/main/proto/ScmAdminProtocol.proto | 31 ++ .../interface-client/src/main/proto/hdds.proto | 16 + .../src/main/proto/ScmServerProtocol.proto | 5 + .../src/main/proto/ScmServerSecurityProtocol.proto | 28 ++ .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 16 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 51 ++- .../hdds/scm/container/ContainerReportHandler.java | 53 +-- .../scm/container/ContainerStateManagerImpl.java | 11 +- .../IncrementalContainerReportHandler.java | 54 +-- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 30 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 22 ++ .../hadoop/hdds/scm/ha/MockSCMHAManager.java | 16 +- .../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 29 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 192 +++++----- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 35 +- .../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 5 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 19 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 94 +++-- .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 1 + .../hdds/scm/node/NodeDecommissionManager.java | 31 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 16 + ...lockLocationProtocolServerSideTranslatorPB.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 49 ++- .../hdds/scm/server/SCMClientProtocolServer.java | 92 +++-- .../hdds/scm/server/SCMDBCheckpointServlet.java | 5 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 33 +- .../hdds/scm/server/StorageContainerManager.java | 143 ++++++-- .../hadoop/hdds/scm/container/MockNodeManager.java | 21 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../TestIncrementalContainerReportHandler.java | 96 ++++- .../algorithms/TestContainerPlacementFactory.java | 51 ++- .../TestSCMContainerPlacementCapacity.java | 37 +- .../TestSCMContainerPlacementRackAware.java | 105 ++++-- .../TestSCMContainerPlacementRandom.java | 32 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 76 ++++ .../hdds/scm/node/TestNodeDecommissionManager.java | 33 ++ .../scm/pipeline/TestRatisPipelineProvider.java | 4 +- .../hdds/scm/cli/ContainerOperationClient.java | 30 +- .../apache/hadoop/ozone/client/ObjectStore.java | 4 + .../apache/hadoop/ozone/client/OzoneBucket.java | 121 ++++--- .../hadoop/ozone/client/OzoneKeyLocation.java | 15 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 57 ++- .../client/io/BlockOutputStreamEntryPool.java | 2 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 5 +- .../ozone/client/protocol/ClientProtocol.java | 65 ++++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 119 +++++-- .../hadoop/ozone/client/MockBlockAllocator.java} | 36 +- .../hadoop/ozone/client/MockDatanodeStorage.java | 73 ++++ .../hadoop/ozone/client/MockOmTransport.java | 276 +++++++++++++++ .../ozone/client/MockXceiverClientFactory.java | 72 ++++ .../hadoop/ozone/client/MockXceiverClientSpi.java | 180 ++++++++++ .../ozone/client/SinglePipelineBlockAllocator.java | 90 +++++ .../hadoop/ozone/client/TestOzoneClient.java | 174 +++++++++ .../main/java/org/apache/hadoop/ozone/OFSPath.java | 3 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../org/apache/hadoop/ozone/audit/OMAction.java | 3 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 39 +-- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 ++ .../ozone/om/protocol/OzoneManagerProtocol.java | 8 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 11 + .../ozone/security/acl/IAccessAuthorizer.java | 1 - hadoop-ozone/dev-support/checks/_lib.sh | 2 +- hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/dependency.sh | 50 +++ hadoop-ozone/dev-support/checks/junit.sh | 2 +- .../src/main/compose/ozone-ha/docker-compose.yaml | 10 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 16 +- .../src/main/compose/ozonesecure-ha/docker-config | 3 +- .../dist/src/main/compose/ozonesecure-ha/test.sh | 5 + .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../src/main/compose/ozonesecure/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +- hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 235 +++++++++++++ .../test.sh => license/update-jar-report.sh} | 28 +- .../src/main/smoketest/admincli/container.robot | 14 +- .../src/main/smoketest/admincli/datanode.robot | 6 +- .../src/main/smoketest/admincli/pipeline.robot | 7 +- .../smoketest/admincli/replicationmanager.robot | 7 +- .../src/main/smoketest/admincli/safemode.robot | 6 +- .../dist/src/main/smoketest/admincli/scmha.robot | 2 +- .../dist/src/main/smoketest/cli/envvars.robot | 15 + .../dist/src/main/smoketest/commonlib.robot | 11 + .../dist/src/main/smoketest/freon/remove.robot | 43 +++ hadoop-ozone/dist/src/shell/ozone/ozone | 7 +- .../dist/src/shell/ozone/ozone-functions.sh | 3 +- .../hdds/scm/TestSCMDbCheckpointServlet.java | 9 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 12 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +- .../org/apache/hadoop/ozone/RatisTestHelper.java | 33 ++ .../hadoop/ozone/TestContainerOperations.java | 21 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 78 ++++- .../hadoop/ozone/TestStorageContainerManager.java | 32 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 3 +- .../TestContainerStateMachineFailureOnRead.java | 6 +- .../client/rpc/TestDeleteWithSlowFollower.java | 10 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 24 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 12 - .../ozone/client/rpc/TestWatchForCommit.java | 3 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 11 +- .../ozoneimpl/TestSecureOzoneContainer.java | 151 ++++---- .../server/TestSecureContainerServer.java | 234 ++++++++----- .../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 243 +++++++++++++ .../TestDatanodeHddsVolumeFailureDetection.java | 289 +++++++++++++++ .../ozone/freon/TestFreonWithDatanodeRestart.java | 126 ------- .../ozone/freon/TestHadoopDirTreeGenerator.java | 2 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 5 +- .../ozone/freon/TestOzoneClientKeyGenerator.java | 94 ----- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 19 + .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 274 +++++++++------ .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 73 ++-- .../src/main/proto/OmClientProtocol.proto | 8 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 27 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 95 +++-- .../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../om/request/bucket/OMBucketDeleteRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 9 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../om/request/s3/security/S3GetSecretRequest.java | 14 +- .../request/s3/security/S3RevokeSecretRequest.java | 155 ++++++++ .../s3/security/S3RevokeSecretResponse.java | 57 +++ ...OzoneManagerProtocolServerSideTranslatorPB.java | 27 +- .../security/OzoneBlockTokenSecretManager.java | 86 +---- .../OzoneDelegationTokenSecretManager.java | 1 + .../security/TestOzoneBlockTokenSecretManager.java | 285 ++++++++------- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 32 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 32 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 - hadoop-ozone/ozonefs-hadoop3/pom.xml | 10 - hadoop-ozone/ozonefs/pom.xml | 10 - hadoop-ozone/recon/pom.xml | 2 +- .../ozone/recon/scm/ReconContainerManager.java | 107 ++++-- .../recon/scm/ReconContainerReportHandler.java | 24 +- .../ReconIncrementalContainerReportHandler.java | 2 +- .../recon/spi/StorageContainerServiceProvider.java | 8 + .../impl/StorageContainerServiceProviderImpl.java | 6 + .../ozone/recon/api/TestContainerEndpoint.java | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 8 + .../scm/AbstractReconContainerManagerTest.java | 30 ++ .../ozone/recon/scm/TestReconContainerManager.java | 50 ++- ...TestReconIncrementalContainerReportHandler.java | 3 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 17 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 283 +++++++++++++-- .../s3/endpoint/ListMultipartUploadsResult.java | 53 +-- .../endpoint/PutBucketAclRequestUnmarshaller.java | 85 +++++ .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 342 ++++++++++++++++++ .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 235 +++++++++++++ .../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 72 ++++ .../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 + .../hadoop/ozone/client/OzoneBucketStub.java | 28 +- .../hadoop/ozone/client/OzoneVolumeStub.java | 24 ++ .../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +- .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 266 ++++++++++++++ .../{TestBucketGet.java => TestBucketList.java} | 76 ++-- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 6 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 56 ++- .../src/test/resources/groupAccessControlList.xml | 57 +++ .../src/test/resources/userAccessControlList.xml | 42 +++ .../org/apache/hadoop/ozone/debug/OzoneDebug.java | 10 + .../ContainerCommands.java} | 130 +++---- .../ozone/debug/container/ExportSubcommand.java | 75 ++++ .../ozone/debug/container/InfoSubcommand.java | 53 +++ .../container/ListSubcommand.java} | 53 ++- .../hadoop/ozone/debug/container/package-info.java | 22 ++ .../hadoop/ozone/freon/BaseFreonGenerator.java | 7 + .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +- .../hadoop/ozone/freon/OmBucketGenerator.java | 2 +- ...OmBucketGenerator.java => OmBucketRemover.java} | 36 +- ...etGenerator.java => OzoneClientKeyRemover.java} | 60 ++-- .../hadoop/ozone/freon/RandomKeyGenerator.java | 137 ++++++++ .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 + .../hadoop/ozone/freon/StreamingGenerator.java | 144 ++++++++ .../containergenerator/GeneratorDatanode.java | 2 +- .../GenerateOzoneRequiredConfigurations.java | 44 ++- .../ozone/genesis/BenchmarkChunkManager.java | 4 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 28 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 27 +- .../hadoop/ozone/shell/s3/GetS3SecretHandler.java | 11 +- ...cretHandler.java => RevokeS3SecretHandler.java} | 57 +-- .../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 3 +- .../TestGenerateOzoneRequiredConfigurations.java | 52 ++- pom.xml | 59 +--- 315 files changed, 11476 insertions(+), 3462 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
