This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-6517-Snapshot in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a0a9d221d022bd4ce396e03e186d71e8914cbe23 Merge: 2410551d2e 8e11723984 Author: Nandakumar <[email protected]> AuthorDate: Mon Aug 8 20:15:27 2022 +0530 Merge branch 'master' into HDDS-6517-Snapshot CONTRIBUTING.md | 146 ++-- .../annotations/ReplicateAnnotationProcessor.java | 77 ++ .../services/javax.annotation.processing.Processor | 3 +- dev-support/ci/selective_ci_checks.bats | 12 + dev-support/ci/selective_ci_checks.sh | 10 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 10 +- .../hadoop/hdds/scm/XceiverClientManager.java | 2 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 13 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 32 +- .../hdds/scm/storage/ECBlockOutputStream.java | 1 + .../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +- .../io/ECBlockReconstructedStripeInputStream.java | 13 +- .../hadoop/hdds/client/ReplicationConfig.java | 2 +- .../hdds/client/ReplicationConfigValidator.java | 4 +- .../hdds/scm/PlacementPolicyValidateProxy.java | 51 ++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 + .../apache/hadoop/hdds/scm/client/ScmClient.java | 7 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 + .../protocol/StorageContainerLocationProtocol.java | 9 + .../hdds/scm/storage/ContainerProtocolCalls.java | 4 +- .../hadoop/hdds/utils/BackgroundService.java | 9 + .../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 + .../org/apache/hadoop/ozone/audit/AuditLogger.java | 6 + .../org/apache/hadoop/ozone/audit/AuditMarker.java | 3 +- .../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +- .../upgrade/AbstractLayoutVersionManager.java | 17 +- .../ozone/upgrade/BasicUpgradeFinalizer.java | 59 +- .../DefaultUpgradeFinalizationExecutor.java | 6 +- .../common/src/main/resources/ozone-default.xml | 8 + .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 17 + .../ozone/container/ContainerTestHelper.java | 65 +- .../InjectedUpgradeFinalizationExecutor.java | 3 - .../ozone/upgrade/TestBasicUpgradeFinalizer.java | 105 ++- .../hadoop/ozone/upgrade/UpgradeTestUtils.java | 59 ++ .../hadoop/hdds/conf/ConfigurationSource.java | 8 + .../helpers/BlockDeletingServiceMetrics.java | 102 +++ .../ozone/container/common/impl/ContainerSet.java | 80 +- .../container/common/impl/HddsDispatcher.java | 25 +- .../common/statemachine/DatanodeConfiguration.java | 17 + .../common/utils/db/DatanodeDBProfile.java | 26 +- .../volume/CapacityVolumeChoosingPolicy.java | 6 +- .../volume/RoundRobinVolumeChoosingPolicy.java | 6 +- .../ozone/container/common/volume/VolumeInfo.java | 33 +- .../ozone/container/common/volume/VolumeUsage.java | 9 +- .../reconstruction/ECContainerOperationClient.java | 40 + .../ECReconstructionCoordinator.java | 61 +- .../ECReconstructionCoordinatorTask.java | 6 + .../container/ec/reconstruction/TokenHelper.java | 6 +- .../container/keyvalue/KeyValueContainerCheck.java | 10 +- .../container/keyvalue/KeyValueContainerData.java | 1 + .../ozone/container/keyvalue/KeyValueHandler.java | 24 +- .../container/keyvalue/helpers/ChunkUtils.java | 50 +- .../background/BlockDeletingService.java | 33 +- .../StaleRecoveringContainerScrubbingService.java | 93 +++ .../container/metadata/AbstractDatanodeStore.java | 12 +- .../metadata/DatanodeSchemaOneDBDefinition.java | 9 +- .../metadata/DatanodeSchemaThreeDBDefinition.java | 5 +- .../ozone/container/ozoneimpl/ContainerReader.java | 15 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 83 +- .../upgrade/DataNodeUpgradeFinalizer.java | 2 + .../protocol/StorageContainerDatanodeProtocol.java | 3 +- .../protocol/commands/DeleteContainerCommand.java | 20 +- .../commands/ReconstructECContainersCommand.java | 15 +- .../commands/ReplicateContainerCommand.java | 20 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +- .../hadoop/ozone/TestHddsDatanodeService.java | 11 + .../container/common/TestBlockDeletingService.java | 52 +- .../common/TestKeyValueContainerData.java | 7 + .../TestSchemaOneBackwardsCompatibility.java | 2 +- .../TestSchemaTwoBackwardsCompatibility.java | 2 +- ...stStaleRecoveringContainerScrubbingService.java | 181 +++++ .../impl/TestContainerDeletionChoosingPolicy.java | 4 +- .../common/impl/TestContainerPersistence.java | 2 +- .../container/common/impl/TestContainerSet.java | 8 +- .../container/common/impl/TestHddsDispatcher.java | 4 +- .../TestCloseContainerCommandHandler.java | 2 +- .../states/endpoint/TestHeartbeatEndpointTask.java | 8 +- .../common/volume/TestReservedVolumeSpace.java | 149 ++++ .../container/common/volume/TestVolumeSet.java | 4 +- .../container/keyvalue/TestKeyValueHandler.java | 4 +- .../container/keyvalue/helpers/TestChunkUtils.java | 22 +- .../container/ozoneimpl/TestContainerReader.java | 26 +- .../ReplicationSupervisorScheduling.java | 2 +- .../replication/TestReplicationSupervisor.java | 2 +- .../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +- .../TestReconstructionECContainersCommands.java | 16 +- hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 69 ++ .../docs/content/feature/S3-Multi-Tenancy-Setup.md | 27 +- hadoop-hdds/docs/content/feature/Topology.md | 4 +- hadoop-hdds/docs/content/interface/ReconApi.md | 5 +- .../scm/protocol/ScmBlockLocationProtocol.java | 3 +- ...inerLocationProtocolClientSideTranslatorPB.java | 13 + .../certificate/authority/CertificateStore.java | 6 +- .../certificate/authority/DefaultCAServer.java | 8 +- .../hdds/utils/db/DBColumnFamilyDefinition.java | 8 +- .../hadoop/hdds/utils/db/DBConfigFromFile.java | 8 +- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 46 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 73 +- .../hadoop/hdds/utils/db/RDBBatchOperation.java | 12 +- .../hadoop/hdds/utils/db/RDBSstFileLoader.java | 10 +- .../hadoop/hdds/utils/db/RDBSstFileWriter.java | 14 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 50 +- .../hadoop/hdds/utils/db/RDBStoreIterator.java | 30 +- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 131 ++-- .../apache/hadoop/hdds/utils/db/TableConfig.java | 13 +- .../hdds/utils/db/managed/ManagedBloomFilter.java | 27 +- .../hdds/utils/db/managed/ManagedCheckpoint.java | 33 + .../db/managed/ManagedColumnFamilyOptions.java | 48 ++ .../hdds/utils/db/managed/ManagedDBOptions.java | 27 +- .../hdds/utils/db/managed/ManagedEnvOptions.java | 28 +- .../hdds/utils/db/managed/ManagedFlushOptions.java | 27 +- .../managed/ManagedIngestExternalFileOptions.java | 28 +- .../hdds/utils/db/managed/ManagedLRUCache.java | 30 +- .../hdds/utils/db/managed/ManagedObject.java | 40 +- .../hdds/utils/db/managed/ManagedOptions.java | 27 +- .../hdds/utils/db/managed/ManagedReadOptions.java | 30 +- .../hdds/utils/db/managed/ManagedRocksDB.java | 81 ++ .../utils/db/managed/ManagedRocksIterator.java | 34 + .../db/managed/ManagedRocksObjectMetrics.java | 60 ++ .../utils/db/managed/ManagedRocksObjectUtils.java | 52 ++ .../utils/db/managed/ManagedSstFileWriter.java | 33 +- .../hdds/utils/db/managed/ManagedStatistics.java | 28 +- .../db/managed/ManagedTransactionLogIterator.java | 30 +- .../hdds/utils/db/managed/ManagedWriteBatch.java | 33 +- .../hdds/utils/db/managed/ManagedWriteOptions.java | 30 +- .../hadoop/hdds/utils/db/managed/package-info.java | 35 + .../security/token/TestBlockTokenVerifier.java | 3 +- .../security/token/TestContainerTokenVerifier.java | 3 +- .../token/TestOzoneBlockTokenSecretManager.java | 51 +- .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 8 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 18 +- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 15 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 13 +- .../src/main/proto/ScmAdminProtocol.proto | 12 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 44 +- .../apache/hadoop/hdds/scm/block/BlockManager.java | 3 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 16 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 30 +- .../scm/block/DeletedBlockLogStateManager.java | 11 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 33 + .../container/AbstractContainerReportHandler.java | 10 +- .../scm/container/CloseContainerEventHandler.java | 4 +- .../hdds/scm/container/ContainerManager.java | 8 +- .../hdds/scm/container/ContainerManagerImpl.java | 11 +- .../hdds/scm/container/ContainerReplica.java | 4 +- .../hdds/scm/container/ContainerReportHandler.java | 4 +- .../hdds/scm/container/ContainerStateManager.java | 7 +- .../scm/container/ContainerStateManagerImpl.java | 42 +- .../IncrementalContainerReportHandler.java | 4 +- .../balancer/AbstractFindTargetGreedy.java | 14 +- .../scm/container/balancer/ContainerBalancer.java | 65 +- .../ContainerBalancerSelectionCriteria.java | 71 +- .../FindTargetGreedyByNetworkTopology.java | 7 +- .../balancer/FindTargetGreedyByUsageInfo.java | 6 +- .../algorithms/SCMContainerPlacementCapacity.java | 7 +- .../algorithms/SCMContainerPlacementRackAware.java | 2 +- .../SCMContainerPlacementRackScatter.java | 16 +- .../algorithms/SCMContainerPlacementRandom.java | 6 +- .../replication/ContainerHealthResult.java | 12 + .../replication/ECContainerReplicaCount.java | 3 +- .../replication/ECOverReplicationHandler.java | 5 + .../replication/ECUnderReplicationHandler.java | 97 ++- .../replication/LegacyReplicationManager.java | 68 +- .../replication/OverReplicatedProcessor.java | 114 +++ .../container/replication/ReplicationManager.java | 175 ++++- .../replication/UnderReplicatedProcessor.java | 125 +++ .../replication/UnhealthyReplicationHandler.java | 9 +- .../hadoop/hdds/scm/ha/BackgroundSCMService.java | 7 +- .../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 4 +- .../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 8 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 18 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 1 + .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 32 +- .../apache/hadoop/hdds/scm/ha/StatefulService.java | 3 +- .../hdds/scm/ha/StatefulServiceStateManager.java | 3 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +- .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 3 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 6 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 3 +- .../hdds/scm/node/StartDatanodeAdminHandler.java | 3 +- .../scm/pipeline/BackgroundPipelineCreator.java | 7 +- .../hdds/scm/pipeline/PipelineActionHandler.java | 3 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 29 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 50 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 6 +- .../hdds/scm/pipeline/PipelineStateManager.java | 13 +- .../scm/pipeline/PipelineStateManagerImpl.java | 14 +- .../scm/pipeline/WritableContainerFactory.java | 3 +- .../scm/pipeline/WritableContainerProvider.java | 4 +- .../scm/pipeline/WritableECContainerProvider.java | 6 +- .../pipeline/WritableRatisContainerProvider.java | 3 +- ...inerLocationProtocolServerSideTranslatorPB.java | 19 + .../scm/safemode/HealthyPipelineSafeModeRule.java | 19 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 22 +- .../hdds/scm/server/SCMClientProtocolServer.java | 136 ++-- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 10 +- .../hdds/scm/server/StorageContainerManager.java | 44 +- .../scm/server/upgrade/FinalizationCheckpoint.java | 25 +- .../scm/server/upgrade/FinalizationManager.java | 16 + .../server/upgrade/FinalizationManagerImpl.java | 38 + .../server/upgrade/FinalizationStateManager.java | 15 +- .../upgrade/FinalizationStateManagerImpl.java | 150 +++- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 88 ++- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 11 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 18 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 58 +- .../container/TestCloseContainerEventHandler.java | 7 +- .../scm/container/TestContainerManagerImpl.java | 7 +- .../scm/container/TestContainerReportHandler.java | 25 +- .../scm/container/TestContainerStateManager.java | 12 +- .../TestIncrementalContainerReportHandler.java | 14 +- .../container/balancer/TestContainerBalancer.java | 120 +-- .../container/balancer/TestFindTargetStrategy.java | 3 +- .../TestSCMContainerPlacementCapacity.java | 10 + .../TestSCMContainerPlacementRackAware.java | 5 +- .../TestSCMContainerPlacementRackScatter.java | 6 +- .../container/replication/ReplicationTestUtil.java | 96 ++- .../replication/TestECOverReplicationHandler.java | 5 + .../replication/TestECUnderReplicationHandler.java | 130 +++- .../replication/TestLegacyReplicationManager.java | 95 ++- .../replication/TestOverReplicatedProcessor.java | 124 +++ .../replication/TestReplicationManager.java | 78 +- .../replication/TestUnderReplicatedProcessor.java | 179 +++++ .../hdds/scm/ha/TestBackgroundSCMService.java | 7 +- .../ha/TestStatefulServiceStateManagerImpl.java | 2 +- .../hdds/scm/node/TestContainerPlacement.java | 3 +- .../hdds/scm/pipeline/MockPipelineManager.java | 16 +- .../TestPipelineDatanodesIntersection.java | 3 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 8 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 14 +- .../scm/pipeline/TestPipelineStateManagerImpl.java | 33 +- .../scm/pipeline/TestRatisPipelineProvider.java | 6 +- .../scm/pipeline/TestSimplePipelineProvider.java | 2 +- .../pipeline/TestWritableECContainerProvider.java | 28 +- .../hdds/scm/update/server/MockCRLStore.java | 4 +- .../server/TestSCMUpdateServiceGrpcServer.java | 6 +- .../hdds/scm/upgrade/TestScmFinalization.java | 52 +- .../org/apache/ozone/test/GenericTestUtils.java | 7 - .../hdds/scm/cli/ContainerOperationClient.java | 5 + .../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++--- .../apache/hadoop/ozone/client/OzoneClient.java | 13 +- .../hadoop/ozone/client/OzoneKeyDetails.java | 20 +- .../checksum/AbstractBlockChecksumComputer.java | 5 +- .../checksum/ReplicatedBlockChecksumComputer.java | 82 +- .../checksum/ReplicatedFileChecksumHelper.java | 16 +- .../ozone/client/io/ECBlockOutputStreamEntry.java | 32 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 30 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 19 +- .../hadoop/ozone/client/TestOzoneECClient.java | 72 +- .../TestReplicatedBlockChecksumComputer.java | 56 +- hadoop-ozone/common/pom.xml | 6 - .../main/java/org/apache/hadoop/ozone/OmUtils.java | 5 + .../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 102 ++- .../om/ha/HadoopRpcOMFailoverProxyProvider.java | 240 ++++++ ...vider.java => OMFailoverProxyProviderBase.java} | 304 ++------ .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 + .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 7 +- .../hadoop/ozone/om/lock/OzoneManagerLock.java | 18 +- .../hadoop/ozone/om/multitenant/AccessPolicy.java | 2 +- .../hadoop/ozone/om/multitenant/OzoneTenant.java | 10 +- .../ozone/om/multitenant/RangerAccessPolicy.java | 4 +- .../apache/hadoop/ozone/om/multitenant/Tenant.java | 12 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 12 + .../ozone/om/protocolPB/GrpcOmTransport.java | 27 +- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 20 +- .../protocolPB/OMAdminProtocolClientSideImpl.java | 10 +- .../OMInterServiceProtocolClientSideImpl.java | 12 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 18 + .../ozone/om/ha/TestOMFailoverProxyProvider.java | 12 +- .../hadoop/ozone/om/lock/TestKeyPathLock.java | 347 +++++++++ .../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 508 ++++-------- .../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 42 +- .../dist/src/main/compose/ozone/monitoring.conf | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 14 - .../src/main/compose/ozonesecure/docker-config | 26 +- .../ozonesecure/mockserverInitialization.json | 98 --- .../main/compose/upgrade/compose/ha/docker-config | 1 + hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +- .../dist/src/main/smoketest/s3/objecthead.robot | 47 ++ .../src/main/smoketest/s3/s3_compatbility_check.sh | 1 + .../smoketest/security/ozone-secure-tenant.robot | 4 + .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 13 +- .../network-tests/src/test/blockade/README.md | 4 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 54 ++ .../TestContainerStateManagerIntegration.java | 2 +- .../metrics/TestSCMContainerManagerMetrics.java | 2 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 3 +- .../hdds/scm/pipeline/TestPipelineClose.java | 5 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 4 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 151 +++- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 230 +----- .../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 272 +++++++ .../hadoop/hdds/upgrade/TestScmHAFinalization.java | 352 +++++++++ .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 37 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 28 +- .../TestContainerStateMachineIdempotency.java | 2 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 + .../hadoop/ozone/TestStorageContainerManager.java | 6 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +- .../rpc/TestContainerStateMachineFailures.java | 3 +- .../ozone/client/rpc/TestECKeyOutputStream.java | 55 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 68 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +- .../ozone/container/TestECContainerRecovery.java | 262 +++++++ .../commandhandler/TestBlockDeletion.java | 101 +-- .../transport/server/ratis/TestCSMMetrics.java | 4 +- .../container/metrics/TestContainerMetrics.java | 17 +- .../container/ozoneimpl/TestOzoneContainer.java | 2 +- .../container/server/TestContainerServer.java | 4 +- .../server/TestSecureContainerServer.java | 6 +- .../hadoop/ozone/om/OmFailoverProxyUtil.java | 4 +- .../hadoop/ozone/om/TestListKeysWithFSO.java | 302 ++++---- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 36 +- .../ozone/om/TestOzoneManagerHAWithData.java | 6 +- .../ozone/om/TestOzoneManagerHAWithFailover.java | 6 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +- .../ozone/om/multitenant/RangerUserRequest.java | 278 +++++++ ...estMultiTenantAccessAuthorizerRangerPlugin.java | 269 ------- .../om/multitenant/TestMultiTenantVolume.java | 29 + .../om/multitenant/TestRangerBGSyncService.java | 349 ++++----- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +- .../TestSCMContainerPlacementPolicyMetrics.java | 3 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +- .../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 6 +- .../ozone/scm/TestStorageContainerManagerHA.java | 14 + .../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 5 +- .../hadoop/ozone/shell/TestOzoneTenantShell.java | 27 +- .../TestResetDeletedBlockRetryCountShell.java | 169 ++++ .../src/main/proto/OmClientProtocol.proto | 11 + hadoop-ozone/ozone-manager/pom.xml | 6 - .../hadoop/ozone/om/OMMultiTenantManager.java | 112 ++- .../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 350 +++++---- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 58 +- .../om/hashcodegenerator/OMHashCodeGenerator.java | 20 +- .../StringOMHashCodeGeneratorImpl.java | 16 +- .../ozone/om/hashcodegenerator/package-info.java} | 17 +- .../ozone/om/lock/OBSKeyPathLockStrategy.java | 27 +- .../ozone/om/multitenant/AuthorizerLockImpl.java | 3 +- .../ozone/om/multitenant/CachedTenantState.java | 41 + .../InMemoryMultiTenantAccessController.java | 49 +- .../multitenant/MultiTenantAccessAuthorizer.java | 282 ------- .../MultiTenantAccessAuthorizerDummyPlugin.java | 194 ----- .../MultiTenantAccessAuthorizerRangerPlugin.java | 854 --------------------- .../multitenant/MultiTenantAccessController.java | 190 +++-- .../om/multitenant/OMRangerBGSyncService.java | 194 +++-- .../RangerClientMultiTenantAccessController.java | 261 +++++-- .../RangerRestMultiTenantAccessController.java | 31 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 +- .../metrics/OzoneManagerDoubleBufferMetrics.java | 13 + .../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 7 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 7 +- .../om/request/key/OMOpenKeysDeleteRequest.java | 8 + .../request/s3/security/S3SecretRequestHelper.java | 14 +- .../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +- .../request/s3/tenant/OMTenantCreateRequest.java | 2 + .../protocolPB/OzoneManagerRequestHandler.java | 15 + .../hadoop/ozone/om/TestOMMultiTenantManager.java | 32 +- .../ozone/om/TestOMMultiTenantManagerImpl.java | 126 ++- .../hadoop/ozone/om/failover/TestOMFailovers.java | 11 +- .../TestMultiTenantAccessController.java | 97 ++- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 + .../bucket/TestOMBucketSetPropertyRequest.java | 8 +- .../om/request/key/TestOMKeyCommitRequest.java | 43 ++ .../request/key/TestOMOpenKeysDeleteRequest.java | 71 +- .../s3/security/TestS3GetSecretRequest.java | 11 +- .../volume/TestOMVolumeSetQuotaRequest.java | 8 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +- .../hadoop/fs/ozone/Hadoop27RpcTransport.java | 17 +- .../hadoop/ozone/recon/api/ContainerEndpoint.java | 8 +- .../hadoop/ozone/recon/api/NodeEndpoint.java | 1 + .../ozone/recon/api/types/DatanodeMetadata.java | 14 + .../ozone/recon/fsck/ContainerHealthTask.java | 3 +- .../ozone/recon/scm/ReconContainerManager.java | 7 +- .../ReconIncrementalContainerReportHandler.java | 4 +- .../ozone/recon/scm/ReconPipelineManager.java | 11 +- .../recon/scm/ReconPipelineReportHandler.java | 4 +- .../scm/ReconStorageContainerManagerFacade.java | 3 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 4 +- .../src/views/datanodes/datanodes.tsx | 18 +- .../apache/hadoop/ozone/recon/TestReconCodecs.java | 12 +- .../ozone/recon/api/TestContainerEndpoint.java | 72 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 90 +-- .../ozone/recon/api/TestTaskStatusService.java | 15 +- .../ozone/recon/api/filters/TestAdminFilter.java | 22 +- .../recon/fsck/TestContainerHealthStatus.java | 12 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 16 +- .../TestContainerHealthTaskRecordGenerator.java | 14 +- .../recon/persistence/AbstractReconSqlDBTest.java | 2 + .../TestReconInternalSchemaDefinition.java | 21 +- .../recon/persistence/TestSqlSchemaSetup.java | 6 +- .../persistence/TestStatsSchemaDefinition.java | 29 +- .../TestUtilizationSchemaDefinition.java | 40 +- .../scm/AbstractReconContainerManagerTest.java | 27 +- .../ozone/recon/scm/TestReconContainerManager.java | 42 +- ...TestReconIncrementalContainerReportHandler.java | 16 +- .../ozone/recon/scm/TestReconPipelineManager.java | 5 +- .../recon/scm/TestReconPipelineReportHandler.java | 6 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 21 +- .../TestStorageContainerServiceProviderImpl.java | 12 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 10 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 9 +- .../recon/tasks/TestReconTaskControllerImpl.java | 44 +- .../ozone/recon/tasks/TestTableCountTask.java | 10 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 35 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 131 ++-- .../ozone/s3/signature/AWSSignatureProcessor.java | 30 +- .../hadoop/ozone/s3/signature/AuthOperation.java} | 36 +- .../s3/signature/AuthorizationV2HeaderParser.java | 12 +- .../s3/signature/AuthorizationV4HeaderParser.java | 92 +-- .../s3/signature/AuthorizationV4QueryParser.java | 3 +- .../hadoop/ozone/s3/signature/Credential.java | 14 +- ...Parser.java => MalformedResourceException.java} | 27 +- .../hadoop/ozone/s3/signature/SignatureParser.java | 4 +- .../apache/hadoop/ozone/s3/util/AuditUtils.java | 53 ++ .../hadoop/ozone/client/ClientProtocolStub.java | 573 ++++++++++++++ .../hadoop/ozone/client/ObjectStoreStub.java | 1 + .../hadoop/ozone/client/OzoneBucketStub.java | 6 +- .../hadoop/ozone/client/OzoneClientStub.java | 7 +- .../ozone/protocolPB/TestGrpcOmTransport.java | 28 +- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 + .../ozone/s3/endpoint/TestPermissionCheck.java | 28 +- .../signature/TestAuthorizationV2HeaderParser.java | 70 +- .../signature/TestAuthorizationV4HeaderParser.java | 98 ++- .../s3/signature/TestStringToSignProducer.java | 4 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 5 +- .../ozone/admin/om/UpdateRangerSubcommand.java | 96 +++ .../scm/ResetDeletedBlockRetryCountSubcommand.java | 73 ++ .../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 48 +- .../org/apache/hadoop/ozone/debug/DropTable.java | 6 +- .../ozone/debug/container/ContainerCommands.java | 6 +- .../ozone/freon/ClosedContainerReplicator.java | 33 +- .../hadoop/ozone/freon/OmBucketGenerator.java | 2 +- .../ozone/freon/OmBucketReadWriteKeyOps.java | 14 +- .../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 +- .../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +- .../ozone/freon/OzoneClientKeyGenerator.java | 2 +- .../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 +- .../ozone/freon/OzoneClientKeyValidator.java | 2 +- .../hadoop/ozone/freon/SCMThroughputBenchmark.java | 5 +- pom.xml | 62 +- 457 files changed, 11878 insertions(+), 5997 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
