This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch HDDS-6030 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 70a9b9c160f2779be76026e82358588f8f9457f8 Merge: c48b559 b83c1f9 Author: Doroszlai, Attila <[email protected]> AuthorDate: Thu Mar 3 12:20:02 2022 +0100 Merge remote-tracking branch 'origin/master' into HDDS-6030 .github/workflows/post-commit.yml | 8 +- .gitignore | 3 +- dev-support/ci/selective_ci_checks.bats | 70 +++- dev-support/ci/selective_ci_checks.sh | 76 +++- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 38 ++ .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 2 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 2 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 2 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +- .../java/org/apache/hadoop/hdds/StringUtils.java | 2 +- .../hadoop/hdds/annotation/InterfaceAudience.java | 6 +- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 2 +- .../org/apache/hadoop/hdds/client/QuotaList.java | 11 +- .../hadoop/hdds/client/ReplicationFactor.java | 2 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 2 +- .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 2 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 4 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 2 +- .../RequestTypeDependentRetryPolicyCreator.java | 6 +- .../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +- .../hadoop/hdds/scm/ByteStringConversion.java | 2 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../hadoop/hdds/scm/container/ContainerInfo.java | 21 +- .../scm/container/ReplicationManagerReport.java | 2 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 16 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 28 +- .../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 4 +- .../hadoop/hdds/scm/net/NodeSchemaLoader.java | 8 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 8 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +- .../hadoop/hdds/utils/ResourceSemaphore.java | 10 +- .../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 +- .../hadoop/ozone/audit/AuditEventStatus.java | 2 +- .../org/apache/hadoop/ozone/audit/AuditLogger.java | 4 +- .../apache/hadoop/ozone/audit/AuditLoggerType.java | 2 +- .../org/apache/hadoop/ozone/audit/AuditMarker.java | 4 +- .../apache/hadoop/ozone/audit/AuditMessage.java | 10 +- .../hadoop/ozone/common/ChecksumByteBuffer.java | 2 +- .../ozone/common/IncrementalChunkBuffer.java | 13 +- .../apache/hadoop/ozone/common/StorageInfo.java | 8 +- .../ozone/common/ha/ratis/RatisSnapshotInfo.java | 2 +- .../ozone/container/common/helpers/BlockData.java | 4 +- .../helpers/ContainerCommandRequestPBHelper.java | 10 +- .../java/org/apache/hadoop/ozone/lease/Lease.java | 8 +- .../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +- .../hadoop/ozone/util/ShutdownHookManager.java | 2 +- .../common/src/main/resources/ozone-default.xml | 26 ++ .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 4 +- .../hadoop/hdds/client/TestReplicationConfig.java | 2 +- .../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +- .../ratis/TestContainerCommandRequestMessage.java | 6 +- .../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 4 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 20 +- .../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +- .../hadoop/hdds/utils/MockGatheringChannel.java | 2 +- .../hadoop/hdds/utils/TestResourceSemaphore.java | 6 +- .../org/apache/hadoop/ozone/audit/DummyEntity.java | 2 +- .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +- .../apache/hadoop/ozone/common/TestChecksum.java | 2 +- .../hadoop/ozone/common/TestChunkBuffer.java | 20 +- .../hadoop/ozone/common/TestStateMachine.java | 4 +- .../ozone/container/ContainerTestHelper.java | 70 +--- .../hadoop/ozone/lease/TestLeaseManager.java | 2 +- .../TestDefaultUpgradeFinalizationExecutor.java | 3 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 27 +- .../container/common/helpers/ContainerMetrics.java | 6 +- .../container/common/helpers/ContainerUtils.java | 2 +- .../ozone/container/common/impl/ContainerData.java | 46 +- .../ozone/container/common/impl/ContainerSet.java | 16 +- .../container/common/impl/HddsDispatcher.java | 15 +- .../common/impl/OpenContainerBlockMap.java | 2 +- .../common/interfaces/ContainerInspector.java | 72 ++++ .../ozone/container/common/interfaces/Handler.java | 12 +- .../common/report/ContainerReportPublisher.java | 5 +- .../common/report/IncrementalReportSender.java} | 26 +- .../common/statemachine/DatanodeConfiguration.java | 2 +- .../common/statemachine/DatanodeStateMachine.java | 14 +- .../common/statemachine/StateContext.java | 143 ++++--- .../commandhandler/CommandDispatcher.java | 2 +- .../SetNodeOperationalStateCommandHandler.java | 11 - .../common/states/datanode/InitDatanodeState.java | 9 +- .../states/datanode/RunningDatanodeState.java | 8 +- .../states/endpoint/HeartbeatEndpointTask.java | 23 +- .../states/endpoint/VersionEndpointTask.java | 2 +- .../common/transport/server/ratis/CSMMetrics.java | 2 +- .../server/ratis/ContainerStateMachine.java | 11 +- .../transport/server/ratis/XceiverServerRatis.java | 9 +- .../common/utils/ContainerInspectorUtil.java | 87 ++++ .../container/common/utils/HddsVolumeUtil.java | 2 +- .../container/common/volume/AbstractFuture.java | 13 +- .../container/common/volume/MutableVolumeSet.java | 2 +- .../container/common/volume/StorageVolume.java | 2 +- .../ozone/container/common/volume/VolumeUsage.java | 2 +- .../container/keyvalue/KeyValueContainer.java | 82 +++- .../container/keyvalue/KeyValueContainerCheck.java | 6 +- .../container/keyvalue/KeyValueContainerData.java | 12 +- .../KeyValueContainerMetadataInspector.java | 463 +++++++++++++++++++++ .../ozone/container/keyvalue/KeyValueHandler.java | 101 ++--- .../container/keyvalue/TarContainerPacker.java | 55 ++- .../container/keyvalue/helpers/ChunkUtils.java | 4 + .../helpers/KeyValueContainerLocationUtil.java | 2 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 23 +- .../container/keyvalue/impl/BlockManagerImpl.java | 110 +++-- .../keyvalue/impl/ChunkManagerDispatcher.java | 18 +- .../background/BlockDeletingService.java | 112 +++-- .../container/metadata/AbstractDatanodeStore.java | 8 +- .../container/ozoneimpl/ContainerController.java | 14 +- .../ozoneimpl/ContainerDataScrubberMetrics.java | 6 +- .../ozoneimpl/ContainerMetadataScanner.java | 2 +- .../ContainerMetadataScrubberMetrics.java | 2 +- .../ozone/container/ozoneimpl/ContainerReader.java | 6 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 40 +- .../replication/GrpcReplicationClient.java | 2 +- .../container/replication/MeasuredReplicator.java | 3 +- .../container/stream/DirstreamClientHandler.java | 2 +- .../upgrade/DataNodeUpgradeFinalizer.java | 2 +- .../upgrade/VersionedDatanodeFeatures.java | 2 +- .../commands/RefreshVolumeUsageCommand.java | 57 +++ .../ozone/protocol/commands/ReregisterCommand.java | 2 +- .../hadoop/ozone/container/common/ScmTestMock.java | 42 +- .../container/common/TestBlockDeletingService.java | 13 +- .../ozone/container/common/TestContainerCache.java | 2 +- .../container/common/TestDatanodeStateMachine.java | 5 - .../common/TestKeyValueContainerData.java | 6 +- .../TestSchemaOneBackwardsCompatibility.java | 38 +- .../container/common/helpers/TestBlockData.java | 6 +- .../common/helpers/TestDatanodeVersionFile.java | 14 +- .../common/impl/TestContainerDataYaml.java | 2 +- .../impl/TestContainerDeletionChoosingPolicy.java | 2 +- .../common/impl/TestContainerPersistence.java | 47 --- .../container/common/impl/TestContainerSet.java | 12 +- .../container/common/impl/TestHddsDispatcher.java | 7 +- .../container/common/interfaces/TestHandler.java | 2 +- .../common/report/TestReportPublisher.java | 2 +- .../common/statemachine/TestStateContext.java | 71 +++- .../TestCloseContainerCommandHandler.java | 4 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +- .../common/volume/TestStorageVolumeChecker.java | 12 +- .../container/common/volume/TestVolumeSet.java | 2 +- .../keyvalue/TestKeyValueBlockIterator.java | 24 +- .../container/keyvalue/TestKeyValueContainer.java | 69 ++- .../keyvalue/TestKeyValueContainerCheck.java | 160 +------ ...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++----- .../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++++++ .../container/keyvalue/TestKeyValueHandler.java | 23 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 14 - .../keyvalue/impl/CommonChunkManagerTestCases.java | 5 +- .../keyvalue/impl/TestBlockManagerImpl.java | 57 +-- .../container/ozoneimpl/TestContainerReader.java | 10 +- .../container/ozoneimpl/TestOzoneContainer.java | 6 +- .../replication/TestGrpcOutputStream.java | 8 +- .../replication/TestMeasuredReplicator.java | 15 + .../replication/TestReplicationSupervisor.java | 2 +- .../testutils/BlockDeletingServiceTestImpl.java | 2 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 6 +- hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 + hadoop-hdds/docs/content/concept/Containers.md | 3 +- hadoop-hdds/docs/content/concept/Datanodes.md | 4 +- hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +- hadoop-hdds/docs/content/concept/Recon.md | 9 +- hadoop-hdds/docs/content/feature/OM-HA.md | 4 +- hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 ++- hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +- hadoop-hdds/docs/content/interface/O3fs.md | 2 +- hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +- hadoop-hdds/docs/content/interface/Ofs.md | 2 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 48 ++- .../docs/content/security/SecuringTDE.zh.md | 4 +- hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +- .../docs/dev-support/bin/make_images_responsive.py | 57 +++ .../themes/ozonedoc/layouts/shortcodes/image.html | 2 +- .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 + .../SCMSecurityProtocolClientSideTranslatorPB.java | 39 +- ...inerLocationProtocolClientSideTranslatorPB.java | 4 +- .../scm/update/client/CRLClientUpdateHandler.java | 2 +- .../hdds/scm/update/client/ClientCRLStore.java | 4 +- .../update/client/SCMUpdateServiceGrpcClient.java | 2 +- .../scm/update/client/UpdateServiceConfig.java | 5 +- .../x509/certificate/authority/BaseApprover.java | 2 +- .../certificate/authority/DefaultCAServer.java | 6 +- .../authority/PKIProfiles/DefaultCAProfile.java | 2 +- .../authority/PKIProfiles/DefaultProfile.java | 32 +- ...ateClient.java => CommonCertificateClient.java} | 116 +++--- .../client/DefaultCertificateClient.java | 32 +- .../certificate/client/OMCertificateClient.java | 84 +--- .../client/ReconCertificateClient.java} | 40 +- .../certificates/utils/CertificateSignRequest.java | 10 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 2 +- .../hdds/security/x509/crl/CRLInfoCodec.java | 2 +- ...va => FixedThreadPoolWithAffinityExecutor.java} | 70 +++- .../hadoop/hdds/server/http/ProfileServlet.java | 2 +- .../hadoop/hdds/utils/DBCheckpointMetrics.java | 2 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 5 +- .../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +- .../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +- .../hadoop/hdds/utils/db/DBConfigFromFile.java | 4 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +- .../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 + .../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +- .../hadoop/hdds/utils/db/cache/CacheKey.java | 2 +- .../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +- .../x509/certificate/authority/MockCAStore.java | 2 +- .../client/TestDefaultCertificateClient.java | 2 +- .../x509/certificate/utils/TestCRLCodec.java | 2 +- .../certificates/TestCertificateSignRequest.java | 4 +- .../x509/certificates/TestRootCertificate.java | 2 +- .../security/x509/keys/TestHDDSKeyGenerator.java | 2 +- .../hadoop/hdds/server/events/TestEventQueue.java | 35 +- .../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +- .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 10 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 40 +- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 6 +- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 4 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 2 +- .../hadoop/hdds/utils/db/cache/TestTableCache.java | 20 +- .../src/main/proto/DatanodeClientProtocol.proto | 4 +- .../interface-client/src/main/proto/hdds.proto | 7 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 11 +- .../src/main/proto/ScmServerSecurityProtocol.proto | 8 +- .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +- .../container/AbstractContainerReportHandler.java | 44 +- .../hdds/scm/container/ContainerManager.java | 7 + .../hdds/scm/container/ContainerManagerImpl.java | 78 ++-- .../hdds/scm/container/ContainerReplicaCount.java | 18 +- .../hdds/scm/container/ContainerReportHandler.java | 141 +++++-- .../hdds/scm/container/ContainerStateManager.java | 10 +- .../scm/container/ContainerStateManagerImpl.java | 24 +- .../IncrementalContainerReportHandler.java | 18 +- .../hdds/scm/container/ReplicationManager.java | 24 +- .../balancer/AbstractFindTargetGreedy.java | 6 +- .../scm/container/balancer/ContainerBalancer.java | 121 ++++-- .../balancer/ContainerBalancerConfiguration.java | 27 +- .../balancer/ContainerBalancerMetrics.java | 139 ++++--- .../scm/container/balancer/FindSourceGreedy.java | 6 +- .../ContainerPlacementPolicyFactory.java | 2 +- .../algorithms/SCMContainerPlacementMetrics.java | 2 +- .../algorithms/SCMContainerPlacementRackAware.java | 12 +- .../container/placement/metrics/SCMMetrics.java | 2 +- .../scm/container/states/ContainerStateMap.java | 30 +- .../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 6 +- .../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +- .../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 9 +- .../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +- .../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 4 +- .../hdds/scm/metadata/X509CertificateCodec.java | 2 +- .../apache/hadoop/hdds/scm/node/CommandQueue.java | 2 +- .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +- .../hdds/scm/node/NodeDecommissionManager.java | 32 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 20 +- .../apache/hadoop/hdds/scm/node/NodeStatus.java | 4 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 +- .../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +- .../hdds/scm/node/states/Node2ObjectsMap.java | 2 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 6 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 4 +- .../scm/pipeline/PipelineStateManagerImpl.java | 7 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +- .../scm/pipeline/WritableContainerFactory.java | 2 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 28 ++ ...inerLocationProtocolServerSideTranslatorPB.java | 8 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +- .../hdds/scm/safemode/DataNodeSafeModeRule.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 16 +- .../hdds/scm/server/SCMClientProtocolServer.java | 15 +- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 + .../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +- .../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 18 +- .../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +- .../hdds/scm/server/StorageContainerManager.java | 63 ++- .../scm/server/StorageContainerManagerStarter.java | 2 +- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 37 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 4 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 27 +- .../hdds/scm/container/SimpleMockNodeManager.java | 22 +- .../scm/container/TestContainerManagerImpl.java | 20 +- .../scm/container/TestContainerReportHandler.java | 28 +- .../scm/container/TestContainerStateManager.java | 6 +- .../TestIncrementalContainerReportHandler.java | 30 +- .../hdds/scm/container/TestReplicationManager.java | 134 +++--- .../container/balancer/TestContainerBalancer.java | 96 +++-- .../TestSCMContainerPlacementRackAware.java | 2 +- .../states/TestContainerReplicaCount.java | 5 +- .../hdds/scm/ha/TestReplicationAnnotation.java | 10 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 26 +- .../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 2 +- .../hdds/scm/metadata/TestPipelineIDCodec.java | 2 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 11 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 12 +- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 26 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 13 +- .../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +- .../hdds/scm/node/states/TestNodeStateMap.java | 4 +- .../TestPipelineDatanodesIntersection.java | 4 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 16 +- .../scm/pipeline/TestRatisPipelineProvider.java | 4 +- .../TestOneReplicaPipelineSafeModeRule.java | 8 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 8 +- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +- .../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +-- .../server/TestStorageContainerManagerStarter.java | 6 +- .../server/TestSCMUpdateServiceGrpcServer.java | 24 +- .../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +- .../testutils/ReplicationNodeManagerMock.java | 16 +- .../org/apache/ozone/test/GenericTestUtils.java | 18 +- .../org/apache/ozone/test/LambdaTestUtils.java | 4 +- .../scm/cli/ContainerBalancerStatusSubcommand.java | 2 +- .../cli/ReplicationManagerStatusSubcommand.java | 2 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 2 +- .../hdds/scm/cli/SafeModeExitSubcommand.java | 2 +- .../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +- .../scm/cli/datanode/DecommissionSubCommand.java | 2 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +- .../scm/cli/datanode/MaintenanceSubCommand.java | 4 +- .../scm/cli/datanode/RecommissionSubCommand.java | 2 +- .../scm/cli/container/TestReportSubCommand.java | 12 +- .../scm/cli/datanode/TestListInfoSubcommand.java | 6 +- hadoop-ozone/client/pom.xml | 1 + .../apache/hadoop/ozone/client/ObjectStore.java | 6 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 12 +- .../hadoop/ozone/client/OzoneClientFactory.java | 2 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 2 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 2 +- .../client/checksum/BaseFileChecksumHelper.java | 9 +- .../checksum/ReplicatedFileChecksumHelper.java | 6 +- .../ozone/client/io/BlockOutputStreamEntry.java | 6 +- .../client/io/BlockOutputStreamEntryPool.java | 1 - .../client/io/MultipartCryptoKeyInputStream.java | 4 + .../ozone/client/protocol/ClientProtocol.java | 4 +- .../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 28 +- .../hadoop/ozone/client/TestHddsClientUtils.java | 4 +- .../checksum/TestReplicatedFileChecksumHelper.java | 18 +- .../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 4 +- .../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 +- .../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 + .../hadoop/ozone/om/helpers/OmBucketArgs.java | 10 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 1 + .../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 2 +- .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 4 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 6 +- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 4 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 6 +- .../hadoop/ozone/om/helpers/ServiceInfo.java | 2 +- .../hadoop/ozone/om/helpers/WithObjectID.java | 2 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 21 +- .../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 51 ++- .../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +- .../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +- .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +- .../org/apache/hadoop/ozone/util/RadixTree.java | 2 +- .../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +- .../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +- .../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +- .../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 +- .../ozone/security/TestGDPRSymmetricKey.java | 2 +- .../ozone/security/acl/TestOzoneObjInfo.java | 9 +- .../apache/hadoop/ozone/util/TestRadixTree.java | 4 +- hadoop-ozone/dev-support/checks/_lib.sh | 2 +- hadoop-ozone/dev-support/checks/acceptance.sh | 3 +- hadoop-ozone/dev-support/checks/bats.sh | 8 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 3 + hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/compose/compatibility/.env | 1 + .../main/compose/compatibility/docker-compose.yaml | 2 +- hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 + .../src/main/compose/ozone-csi/docker-compose.yaml | 8 +- hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 + .../src/main/compose/ozone-ha/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone-mr/hadoop27/.env | 1 + .../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 + .../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 + .../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop33/.env | 1 + .../compose/ozone-mr/hadoop33/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-om-ha/.env | 1 + .../dist/src/main/compose/ozone-om-ha/Dockerfile | 3 +- .../dist/src/main/compose/ozone-om-prepare/.env | 6 +- .../compose/ozone-om-prepare/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone-topology/.env | 1 + .../compose/ozone-topology/docker-compose.yaml | 16 +- hadoop-ozone/dist/src/main/compose/ozone/.env | 1 + .../src/main/compose/ozone/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone/freon-ockg.yaml | 2 +- .../dist/src/main/compose/ozone/freon-rk.yaml | 2 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 - .../dist/src/main/compose/ozoneblockade/.env | 1 + .../main/compose/ozoneblockade/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozones3-haproxy/.env | 1 + .../compose/ozones3-haproxy/docker-compose.yaml | 12 +- .../dist/src/main/compose/ozonescripts/.env | 1 + .../dist/src/main/compose/ozonescripts/Dockerfile | 3 +- .../main/compose/ozonescripts/docker-compose.yaml | 3 + .../dist/src/main/compose/ozonesecure-ha/.env | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 22 +- .../dist/src/main/compose/ozonesecure-mr/.env | 1 + .../compose/ozonesecure-mr/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozonesecure-mr/test.sh | 3 - .../dist/src/main/compose/ozonesecure/.env | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 10 +- .../src/main/compose/ozonesecure/docker-config | 4 + .../dist/src/main/compose/ozonesecure/test.sh | 2 +- hadoop-ozone/dist/src/main/compose/restart/.env | 2 +- .../src/main/compose/restart/docker-compose.yaml | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 14 +- .../dist/src/main/compose/upgrade/compose/ha/.env | 1 + .../main/compose/upgrade/compose/ha/docker-config | 4 +- .../src/main/compose/upgrade/compose/non-ha/.env | 1 + .../dist/src/main/compose/upgrade/testlib.sh | 1 - hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 + .../dist/src/main/compose/xcompat/clients.yaml | 20 +- .../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +- hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +- .../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +- .../main/k8s/examples/getting-started/Flekszible | 2 + .../examples/getting-started/config-configmap.yaml | 1 + .../examples/getting-started/kustomization.yaml} | 20 +- .../dist/src/main/k8s/examples/minikube/Flekszible | 2 + .../k8s/examples/minikube/config-configmap.yaml | 1 + .../examples/minikube/kustomization.yaml} | 20 +- .../src/main/k8s/examples/ozone-dev/Flekszible | 1 + .../k8s/examples/ozone-dev/config-configmap.yaml | 1 + .../examples/ozone-dev/kustomization.yaml} | 26 +- .../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 + .../examples/ozone-ha/kustomization.yaml} | 15 +- .../dist/src/main/k8s/examples/ozone/Flekszible | 3 +- .../main/k8s/examples/ozone/config-configmap.yaml | 1 + .../.env => k8s/examples/ozone/kustomization.yaml} | 15 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 3 +- .../src/main/smoketest/compatibility/read.robot | 9 +- .../compatibility/{write.robot => setup.robot} | 16 +- .../src/main/smoketest/compatibility/write.robot | 7 +- .../dist/src/main/smoketest/mapreduce.robot | 1 + .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 1 + .../src/main/smoketest/s3/MultipartUpload.robot | 21 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++ hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 3 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 10 +- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +- .../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +- .../hadoop/ozone/loadgenerators/LoadBucket.java | 2 +- .../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 14 +- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 +- .../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 36 +- .../fs/ozone/contract/ITestOzoneContractUtils.java | 4 +- .../hadoop/fs/ozone/contract/OzoneContract.java | 4 +- .../TestContainerStateManagerIntegration.java | 28 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +- .../TestRatisPipelineCreateAndDestroy.java | 2 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 6 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 6 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 14 +- .../hadoop/ozone/MiniOzoneClusterProvider.java | 4 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +- ...OutputUtil.java => StandardOutputTestBase.java} | 2 +- .../ozone/TestContainerBalancerOperations.java | 4 +- .../hadoop/ozone/TestContainerOperations.java | 2 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +- .../hadoop/ozone/TestMiniOzoneOMHACluster.java | 2 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 15 +- .../hadoop/ozone/TestStorageContainerManager.java | 5 +- .../ozone/client/CertificateClientTestImpl.java | 4 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +- .../client/rpc/TestContainerStateMachine.java | 4 +- .../rpc/TestContainerStateMachineFlushDelay.java | 14 +- .../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +- .../client/rpc/TestFailureHandlingByClient.java | 116 +++++- .../rpc/TestFailureHandlingByClientFlushDelay.java | 5 +- .../rpc/TestMultiBlockWritesWithDnFailures.java | 4 +- .../client/rpc/TestOzoneAtRestEncryption.java | 38 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +- .../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 60 +-- .../rpc/TestOzoneRpcClientForAclAuditLog.java | 21 +- .../TestOzoneRpcClientWithKeyLatestVersion.java | 2 +- .../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +- .../ozone/client/rpc/TestWatchForCommit.java | 2 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 2 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 8 +- .../apache/hadoop/ozone/container/TestHelper.java | 8 +- .../TestCloseContainerByPipeline.java | 2 +- .../commandhandler/TestDeleteContainerHandler.java | 2 +- .../TestRefreshVolumeUsageHandler.java | 145 +++++++ .../transport/server/ratis/TestCSMMetrics.java | 3 +- .../container/metrics/TestContainerMetrics.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 27 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +- .../container/server/TestContainerServer.java | 2 +- .../server/TestSecureContainerServer.java | 16 +- .../TestDatanodeHddsVolumeFailureDetection.java | 2 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 26 +- .../hadoop/ozone/fsck/TestContainerMapper.java | 16 +- .../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +- .../ozone/om/TestContainerReportWithKeys.java | 4 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +- .../apache/hadoop/ozone/om/TestObjectStore.java | 12 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 14 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +- .../ozone/om/TestOmStartupSlvLessThanMlv.java | 2 +- .../ozone/om/TestOzoneManagerConfiguration.java | 4 +- .../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 8 +- .../ozone/om/TestOzoneManagerHAWithData.java | 2 +- .../ozone/om/TestOzoneManagerRestInterface.java | 2 +- .../hadoop/ozone/om/TestOzoneManagerRestart.java | 47 ++- .../hadoop/ozone/om/TestSecureOzoneManager.java | 3 +- .../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +- .../ozone/recon/TestReconWithOzoneManager.java | 16 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 8 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 2 +- .../hadoop/ozone/scm/TestCloseContainer.java | 4 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 10 +- .../scm/node/TestDecommissionAndMaintenance.java | 24 +- .../hadoop/ozone/scm/node/TestQueryNode.java | 2 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +- .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 +-- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 28 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 22 +- .../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 13 +- .../hadoop/ozone/om/OzoneManagerPrepareState.java | 4 +- .../hadoop/ozone/om/OzoneManagerStarter.java | 2 +- .../hadoop/ozone/om/OzonePrefixPathImpl.java | 18 + .../hadoop/ozone/om/S3SecretManagerImpl.java | 2 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +- .../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 6 +- .../hadoop/ozone/om/request/OMClientRequest.java | 21 +- .../om/request/bucket/OMBucketCreateRequest.java | 18 +- .../om/request/bucket/OMBucketDeleteRequest.java | 9 +- .../request/bucket/OMBucketSetPropertyRequest.java | 6 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 2 +- .../om/request/file/OMDirectoryCreateRequest.java | 2 +- .../file/OMDirectoryCreateRequestWithFSO.java | 5 +- .../ozone/om/request/file/OMFileCreateRequest.java | 2 +- .../ozone/om/request/file/OMFileRequest.java | 10 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 4 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 2 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 4 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 8 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +- .../multipart/S3MultipartUploadAbortRequest.java | 2 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 2 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 2 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 2 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 2 +- .../response/key/OMKeyRenameResponseWithFSO.java | 3 +- .../om/response/key/OMKeysRenameResponse.java | 10 + .../multipart/S3MultipartUploadAbortResponse.java | 6 +- .../S3MultipartUploadCommitPartResponse.java | 6 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 2 +- .../protocolPB/OzoneManagerRequestHandler.java | 1 + .../OzoneDelegationTokenSecretManager.java | 2 +- .../hadoop/ozone/security/OzoneSecretStore.java | 4 +- .../hadoop/ozone/om/TestBucketManagerImpl.java | 7 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +- .../apache/hadoop/ozone/om/TestOMDBDefinition.java | 2 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 14 +- .../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +- .../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 4 +- .../om/ratis/TestOzoneManagerRatisServer.java | 2 +- .../om/ratis/TestOzoneManagerStateMachine.java | 2 +- .../bucket/TestOMBucketCreateRequestWithFSO.java | 4 + .../bucket/TestOMBucketDeleteRequestWithFSO.java | 76 ++++ .../request/file/TestOMDirectoryCreateRequest.java | 2 +- .../om/request/file/TestOMFileCreateRequest.java | 6 +- .../om/request/key/TestOMKeyCommitRequest.java | 4 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 2 +- .../om/request/key/TestOMKeyCreateRequest.java | 2 +- .../request/key/TestOMKeyDeleteRequestWithFSO.java | 101 ++++- .../key/TestOMKeyPurgeRequestAndResponse.java | 2 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 2 +- .../TestS3MultipartUploadCompleteRequest.java | 2 +- .../request/volume/TestOMVolumeCreateRequest.java | 2 +- .../volume/TestOMVolumeSetQuotaRequest.java | 2 +- .../ozone/om/response/TestCleanupTableInfo.java | 4 +- .../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 2 +- .../om/upgrade/TestOzoneManagerPrepareState.java | 2 +- .../ozone/security/TestAWSV4AuthValidator.java | 2 +- .../TestOzoneDelegationTokenSecretManager.java | 6 +- .../ozone/security/TestOzoneTokenIdentifier.java | 6 +- .../security/acl/TestOzoneNativeAuthorizer.java | 10 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 9 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 18 + .../fs/ozone/BasicRootedOzoneFileSystem.java | 3 +- .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 + .../apache/hadoop/fs/ozone/OzoneClientUtils.java | 24 +- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 2 +- .../hadoop/fs/ozone/TestOzoneClientUtils.java | 59 +++ hadoop-ozone/ozonefs-shaded/pom.xml | 2 +- .../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 4 +- .../hadoop/ozone/recon/ReconControllerModule.java | 2 + .../org/apache/hadoop/ozone/recon/ReconServer.java | 125 ++++++ .../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 + .../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 ++ .../ozone/recon/api/ClusterStateEndpoint.java | 15 +- .../ozone/recon/api/MetricsProxyEndpoint.java | 4 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 2 +- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +- .../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +- .../ozone/recon/fsck/ContainerHealthTask.java | 4 +- .../recon/metrics/OzoneManagerSyncMetrics.java | 12 + .../recon/metrics/ReconTaskStatusMetrics.java | 83 ++++ .../ozone/recon/scm/ReconContainerManager.java | 4 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +- .../ozone/recon/scm/ReconPipelineFactory.java | 2 +- .../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 ++- .../scm/ReconStorageContainerManagerFacade.java | 11 +- .../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 68 ++- .../impl/StorageContainerServiceProviderImpl.java | 16 +- .../ozone/recon/tasks/FileSizeCountTask.java | 4 +- .../hadoop/ozone/recon/tasks/TableCountTask.java | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 4 + .../ozone/recon/api/TestOpenContainerCount.java | 8 +- .../ozone/recon/api/TestTaskStatusService.java | 2 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 2 +- .../TestUtilizationSchemaDefinition.java | 2 +- .../scm/AbstractReconContainerManagerTest.java | 3 +- .../ozone/recon/scm/TestReconNodeManager.java | 12 +- .../ozone/recon/scm/TestReconPipelineManager.java | 3 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 88 +++- .../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +- .../recon/tasks/TestContainerKeyMapperTask.java | 2 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +- .../ozone/recon/tasks/TestTableCountTask.java | 2 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 19 +- .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 8 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 58 +-- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 24 +- .../ozone/s3/endpoint/ListBucketResponse.java | 6 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 108 +++-- .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +- .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 4 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 20 +- .../hadoop/ozone/s3/signature/Credential.java | 2 +- .../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 39 +- .../ozone/s3/TestVirtualHostStyleFilter.java | 2 +- .../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 2 +- .../ozone/admin/om/FinalizeUpgradeSubCommand.java | 6 +- .../admin/scm/FinalizeScmUpgradeSubcommand.java | 6 +- .../admin/scm/FinalizeUpgradeCommandUtil.java | 4 +- .../hadoop/ozone/audit/parser/AuditParser.java | 2 +- .../ozone/audit/parser/common/DatabaseHelper.java | 16 +- .../parser/handler/TemplateCommandHandler.java | 2 +- .../ozone/audit/parser/model/AuditEntry.java | 22 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 8 +- .../apache/hadoop/ozone/debug/PrefixParser.java | 2 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 19 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 12 +- .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +- .../hadoop/ozone/freon/HadoopFsGenerator.java | 2 +- .../hadoop/ozone/freon/StreamingGenerator.java | 2 +- .../GenerateOzoneRequiredConfigurations.java | 2 +- .../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +- .../ozone/shell/bucket/CreateBucketHandler.java | 2 +- .../hadoop/ozone/audit/parser/TestAuditParser.java | 2 +- .../hadoop/ozone/conf/TestGetConfOptions.java | 4 +- .../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +- .../TestGenerateOzoneRequiredConfigurations.java | 4 +- .../org/apache/hadoop/test/OzoneTestDriver.java | 6 +- pom.xml | 30 +- 719 files changed, 6653 insertions(+), 3426 deletions(-) diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java index 6f563eb,dbf634f..c32965f --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java @@@ -19,14 -17,10 +17,13 @@@ package org.apache.hadoop.hdds.security.x509.certificate.client; - import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; ++import java.io.IOException; + import org.apache.hadoop.hdds.security.x509.SecurityConfig; + import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; + - import java.io.IOException; + import org.slf4j.Logger; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; @@@ -34,36 -28,38 +31,38 @@@ import static org.apache.hadoop.hdds.se import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.SUCCESS; /** - * Certificate client for OzoneManager. + * Common Certificate client. */ - public class OMCertificateClient extends DefaultCertificateClient { - - private static final Logger LOG = - LoggerFactory.getLogger(OMCertificateClient.class); + public class CommonCertificateClient extends DefaultCertificateClient { - public static final String COMPONENT_NAME = "om"; - - public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId, String localCrlId) throws IOException { - super(securityConfig, LOG, certSerialId, COMPONENT_NAME); - this.setLocalCrlId(localCrlId!=null ? - Long.parseLong(localCrlId): 0); - } + private final Logger log; - public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId) throws IOException{ - this(securityConfig, certSerialId, null); + public CommonCertificateClient(SecurityConfig securityConfig, Logger log, - String certSerialId, String component) { ++ String certSerialId, String component) throws IOException { + super(securityConfig, log, certSerialId, component); + this.log = log; } - public OMCertificateClient(SecurityConfig securityConfig) throws IOException { - this(securityConfig, null, null); + /** + * Returns a CSR builder that can be used to creates a Certificate signing + * request. + * + * @return CertificateSignRequest.Builder + */ + @Override + public CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException { + return super.getCSRBuilder() + .setDigitalEncryption(true) + .setDigitalSignature(true); } @Override - protected InitResponse handleCase(InitCase init) throws IOException { + protected InitResponse handleCase(InitCase init) - throws CertificateException { ++ throws CertificateException, IOException { switch (init) { case NONE: - LOG.info("Creating keypair for client as keypair and certificate not " + + log.info("Creating keypair for client as keypair and certificate not " + "found."); bootstrapClientKeys(); return GETCERT; diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 8f88a38,d681806..f5b2405 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@@ -789,9 -686,9 +789,9 @@@ public abstract class DefaultCertificat * */ @Override - public synchronized InitResponse init() throws CertificateException { + public synchronized InitResponse init() throws IOException { int initCase = 0; - PrivateKey pvtKey= getPrivateKey(); + PrivateKey pvtKey = getPrivateKey(); PublicKey pubKey = getPublicKey(); X509Certificate certificate = getCertificate(); diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java index 6f563eb,d6c535a..3ce7d16 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java @@@ -24,15 -23,7 +23,9 @@@ import org.slf4j.Logger import org.slf4j.LoggerFactory; import org.apache.hadoop.hdds.security.x509.SecurityConfig; - import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import java.io.IOException; + - import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; - import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; - import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.RECOVER; - import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.SUCCESS; - /** * Certificate client for OzoneManager. */ @@@ -44,14 -35,14 +37,14 @@@ public class OMCertificateClient extend public static final String COMPONENT_NAME = "om"; public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId, String localCrlId) { + String certSerialId, String localCrlId) throws IOException { super(securityConfig, LOG, certSerialId, COMPONENT_NAME); - this.setLocalCrlId(localCrlId!=null ? - Long.parseLong(localCrlId): 0); + this.setLocalCrlId(localCrlId != null ? + Long.parseLong(localCrlId) : 0); } public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId) throws IOException{ - String certSerialId) { ++ String certSerialId) throws IOException { this(securityConfig, certSerialId, null); } diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java index 12490c5,afbcbf6..482d81a --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java @@@ -15,35 -15,28 +15,31 @@@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hdds.scm.cli; + package org.apache.hadoop.hdds.security.x509.certificate.client; +import java.io.IOException; + - import org.apache.hadoop.hdds.cli.HddsVersionProvider; - import org.apache.hadoop.hdds.scm.client.ScmClient; + import org.apache.hadoop.hdds.security.x509.SecurityConfig; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import picocli.CommandLine.Command; /** - * This is the handler that process safe mode exit command. + * Certificate client for Recon. */ - @Command( - name = "exit", - description = "Force SCM out of safe mode", - mixinStandardHelpOptions = true, - versionProvider = HddsVersionProvider.class) - public class SafeModeExitSubcommand extends ScmSubcommand { - + public class ReconCertificateClient extends CommonCertificateClient { private static final Logger LOG = - LoggerFactory.getLogger(SafeModeExitSubcommand.class); + LoggerFactory.getLogger(ReconCertificateClient.class); + + public static final String COMPONENT_NAME = "recon"; + + public ReconCertificateClient(SecurityConfig securityConfig, - String certSerialId) { ++ String certSerialId) throws IOException { + super(securityConfig, LOG, certSerialId, COMPONENT_NAME); + } @Override - public void execute(ScmClient scmClient) throws IOException { - boolean execReturn = scmClient.forceExitSafeMode(); - if(execReturn){ - LOG.info("SCM exit safe mode successfully."); - } + public Logger getLogger() { + return LOG; } } diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index ede522d,a4171e8..60f1099 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@@ -301,13 -299,7 +315,13 @@@ public class SCMSecurityProtocolServer */ @Override public String getCACertificate() throws IOException { - LOGGER.debug("Getting CA certificate."); + if (securityConfig.isCustomCAEnabled()) { + throw new SCMSecurityException("Get CA Certificate is not supported " + + "when custom CA is enabled."); + } - if(LOGGER.isDebugEnabled()) { ++ if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Getting CA certificate."); + } try { return CertificateCodec.getPEMEncodedString( scmCertificateServer.getCACertificate()); diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 8d8abdd,05ac12c..7eb5a3d --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -809,10 -817,10 +833,10 @@@ public final class StorageContainerMana * @param configurator - configurator * @throws IOException - on Failure */ - private void initalizeMetadataStore(OzoneConfiguration conf, - SCMConfigurator configurator) + private void initializeMetadataStore(OzoneConfiguration conf, + SCMConfigurator configurator) throws IOException { - if(configurator.getMetadataStore() != null) { + if (configurator.getMetadataStore() != null) { scmMetadataStore = configurator.getMetadataStore(); } else { scmMetadataStore = new SCMMetadataStoreImpl(conf); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
