This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-6517-Snapshot in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4be3864675ad98801cda5caabde29734420bc662 Merge: 6eb19a7710 4e364001ac Author: Nandakumar <[email protected]> AuthorDate: Thu Oct 13 08:56:23 2022 +0530 Merge branch 'master' into HDDS-6517-Snapshot .github/workflows/close-pending.yaml | 2 +- .github/workflows/comments.yaml | 2 +- .github/workflows/post-commit.yml | 26 +- dev-support/annotations/pom.xml | 114 -- .../services/javax.annotation.processing.Processor | 17 - dev-support/ci/pr_title_check.bats | 8 + dev-support/ci/pr_title_check.sh | 2 + hadoop-hdds/annotations/pom.xml | 34 + .../annotations/ReplicateAnnotationProcessor.java | 3 + .../RequestFeatureValidatorProcessor.java | 0 .../org/apache/ozone/annotations/package-info.java | 0 hadoop-hdds/client/pom.xml | 2 +- .../hadoop/hdds/scm/ContainerClientMetrics.java | 138 +++ .../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 10 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 3 +- .../hdds/scm/storage/ECBlockOutputStream.java | 6 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 7 +- .../hadoop/ozone/client/io/ECBlockInputStream.java | 2 +- .../client/io/ECBlockReconstructedInputStream.java | 2 +- .../hdds/scm/TestContainerClientMetrics.java | 116 ++ .../storage/TestBlockOutputStreamCorrectness.java | 4 +- .../hdds/scm/storage/TestChunkInputStream.java | 9 +- .../ozone/client/io/TestECBlockInputStream.java | 3 +- hadoop-hdds/common/pom.xml | 12 +- .../hdds/client/DefaultReplicationConfig.java | 2 +- .../hadoop/hdds/client/ECReplicationConfig.java | 2 +- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 9 - .../org/apache/hadoop/hdds/client/QuotaList.java | 19 +- .../hadoop/hdds/client/RatisReplicationConfig.java | 2 +- .../hadoop/hdds/client/ReplicationConfig.java | 4 +- .../hdds/client/ReplicationConfigValidator.java | 32 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 11 + .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 5 +- .../apache/hadoop/hdds/scm/PlacementPolicy.java | 18 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +- .../protocol/StorageContainerLocationProtocol.java | 2 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 + .../org/apache/hadoop/ozone/common/Checksum.java | 3 +- .../services/javax.annotation.processing.Processor | 2 + .../common/src/main/resources/ozone-default.xml | 73 +- .../hdds/client/TestECReplicationConfig.java | 29 +- .../hadoop/hdds/client/TestReplicationConfig.java | 36 +- .../client/TestReplicationConfigValidator.java | 14 +- .../hadoop/hdds/conf/SimpleConfiguration.java | 13 + .../hadoop/hdds/conf/TestOzoneConfiguration.java | 10 + .../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 + .../hadoop/hdds/conf/ConfigFileGenerator.java | 3 + .../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +- .../hdds/conf/ConfigurationReflectionUtil.java | 9 + .../hadoop/hdds/conf/ConfigurationTarget.java | 4 + .../hadoop/hdds/conf/ConfigurationExample.java | 13 + .../hdds/conf/TestConfigurationReflectionUtil.java | 4 + .../dev-support/findbugsExcludeFile.xml | 2 +- hadoop-hdds/container-service/pom.xml | 2 +- .../datanode/metadata/DatanodeCRLStoreImpl.java | 13 +- .../container/common/report/ReportManager.java | 4 +- .../container/common/report/ReportPublisher.java | 6 +- .../common/report/ReportPublisherFactory.java | 6 +- .../common/statemachine/DatanodeConfiguration.java | 107 ++ .../common/statemachine/EndpointStateMachine.java | 16 +- .../statemachine/EndpointStateMachineMBean.java | 2 + .../common/statemachine/StateContext.java | 56 +- .../commandhandler/DeleteBlocksCommandHandler.java | 32 +- .../DeleteContainerCommandHandler.java | 11 +- .../states/endpoint/HeartbeatEndpointTask.java | 6 +- .../container/common/utils/HddsVolumeUtil.java | 8 +- .../common/utils/db/DatanodeDBProfile.java | 21 +- .../common/volume/AvailableSpaceFilter.java | 84 ++ .../volume/CapacityVolumeChoosingPolicy.java | 42 +- .../ozone/container/common/volume/HddsVolume.java | 14 +- .../volume/RoundRobinVolumeChoosingPolicy.java | 22 +- .../common/volume/VolumeChoosingUtil.java | 49 + .../container/common/volume/VolumeIOStats.java | 10 +- .../container/common/volume/VolumeInfoMetrics.java | 5 + .../ECReconstructionCoordinator.java | 8 +- .../container/keyvalue/KeyValueContainer.java | 9 +- .../keyvalue/impl/ChunkManagerFactory.java | 12 +- .../background/BlockDeletingService.java | 27 +- .../container/metadata/AbstractDatanodeStore.java | 40 +- .../metadata/DatanodeSchemaThreeDBDefinition.java | 2 +- .../ozoneimpl/AbstractContainerScanner.java | 136 +++ ...s.java => AbstractContainerScannerMetrics.java} | 42 +- .../container/ozoneimpl/ContainerDataScanner.java | 135 +-- .../ozoneimpl/ContainerDataScannerMetrics.java | 69 ++ .../ozoneimpl/ContainerDataScrubberMetrics.java | 120 -- .../ozoneimpl/ContainerMetadataScanner.java | 112 +- .../ozoneimpl/ContainerMetadataScannerMetrics.java | 44 + ...ion.java => ContainerScannerConfiguration.java} | 12 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 9 +- .../upgrade/DatanodeSchemaV3FinalizeAction.java | 3 +- .../protocol/commands/CommandForDatanode.java | 4 +- .../hadoop/ozone/protocol/commands/SCMCommand.java | 4 +- .../webapps/hddsDatanode/dn-overview.html | 2 + .../common/report/TestReportPublisher.java | 6 +- .../common/statemachine/TestStateContext.java | 22 +- .../container/common/utils/TestHddsVolumeUtil.java | 8 +- .../volume/TestCapacityVolumeChoosingPolicy.java | 21 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 33 +- .../TestVolumeIOStatsWithPrometheusSink.java | 92 ++ .../container/keyvalue/TestKeyValueContainer.java | 36 - .../keyvalue/TestKeyValueContainerCheck.java | 15 +- ...java => TestContainerScannerConfiguration.java} | 28 +- ...trics.java => TestContainerScannerMetrics.java} | 20 +- hadoop-hdds/docs/content/concept/Recon.md | 2 +- hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 12 +- .../docs/content/security/SecuringTDE.zh.md | 6 +- hadoop-hdds/framework/pom.xml | 16 +- ...inerLocationProtocolClientSideTranslatorPB.java | 2 +- .../org/apache/hadoop/hdds/server/OzoneAdmins.java | 87 ++ .../hadoop/hdds/server/http/BaseHttpServer.java | 8 +- .../hadoop/hdds/server/http/HttpServer2.java | 3 + .../hdds/server/http/HttpServer2Metrics.java | 94 ++ .../hadoop/hdds/utils/DBCheckpointServlet.java | 11 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 30 +- .../apache/hadoop/hdds/utils/db/CodecRegistry.java | 22 +- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 19 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 57 +- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 7 +- .../hadoop/hdds/utils/db/RDBSstFileWriter.java | 18 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 25 +- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 45 +- .../apache/hadoop/hdds/utils/db/TableConfig.java | 14 +- .../db/managed/ManagedBlockBasedTableConfig.java | 74 ++ .../db/managed/ManagedColumnFamilyOptions.java | 37 +- .../utils/db/managed/ManagedRocksObjectUtils.java | 2 +- ...dColumnFamilyOptions.java => ManagedSlice.java} | 28 +- .../server/http/TestHttpServer2MetricsTest.java | 94 ++ hadoop-hdds/hadoop-dependency-client/pom.xml | 14 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 24 + hadoop-hdds/pom.xml | 7 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 51 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 12 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 6 - .../hdds/scm/container/ContainerReplica.java | 2 +- .../scm/container/ContainerStateManagerImpl.java | 178 +-- .../scm/container/balancer/ContainerBalancer.java | 179 +-- .../balancer/ContainerBalancerMetrics.java | 29 + .../algorithms/SCMContainerPlacementCapacity.java | 12 +- .../algorithms/SCMContainerPlacementRackAware.java | 14 +- .../SCMContainerPlacementRackScatter.java | 264 +++-- .../algorithms/SCMContainerPlacementRandom.java | 13 +- .../replication/ContainerCheckRequest.java | 120 ++ .../replication/ContainerHealthCheck.java | 35 - .../replication/ContainerHealthResult.java | 9 +- .../replication/ECContainerHealthCheck.java | 86 -- .../replication/ECContainerReplicaCount.java | 48 +- .../replication/ECOverReplicationHandler.java | 22 +- .../replication/ECUnderReplicationHandler.java | 191 ++- .../replication/LegacyReplicationManager.java | 4 +- .../replication/OverReplicatedProcessor.java | 37 +- .../container/replication/ReplicationManager.java | 252 ++-- .../container/replication/ReplicationQueue.java | 73 ++ .../replication/UnderReplicatedProcessor.java | 37 +- .../replication/health/AbstractCheck.java | 71 ++ .../ClosedWithMismatchedReplicasHandler.java | 89 ++ .../health/ClosingContainerHandler.java | 62 + .../health/ECReplicationCheckHandler.java | 134 +++ .../container/replication/health/HealthCheck.java | 56 + .../replication/health/OpenContainerHandler.java | 81 ++ .../health/QuasiClosedContainerHandler.java | 131 ++ .../container/replication/health/package-info.java | 10 +- .../scm/container/states/ContainerAttribute.java | 14 +- .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 + .../apache/hadoop/hdds/scm/ha/StatefulService.java | 12 +- .../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 11 +- .../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 2 +- .../hdds/scm/ha/io/GeneratedMessageCodec.java | 12 +- .../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 2 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 51 +- .../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 116 ++ .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 10 +- .../hadoop/hdds/scm/server/SCMCertStore.java | 5 +- .../hdds/scm/server/SCMClientProtocolServer.java | 2 +- .../hdds/scm/server/SCMDBCheckpointServlet.java | 1 + .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 4 +- .../hdds/scm/server/StorageContainerManager.java | 52 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 154 ++- .../container/balancer/TestContainerBalancer.java | 189 +-- .../algorithms/TestContainerPlacementFactory.java | 4 +- .../TestSCMContainerPlacementRackScatter.java | 76 +- .../container/replication/ReplicationTestUtil.java | 56 +- .../replication/TestECContainerReplicaCount.java | 84 +- .../replication/TestECOverReplicationHandler.java | 5 +- .../replication/TestECUnderReplicationHandler.java | 100 +- .../replication/TestOverReplicatedProcessor.java | 7 +- .../replication/TestReplicationManager.java | 102 +- .../replication/TestUnderReplicatedProcessor.java | 7 +- .../TestClosedWithMismatchedReplicasHandler.java | 217 ++++ .../health/TestClosingContainerHandler.java | 208 ++++ .../TestECReplicationCheckHandler.java} | 199 +++- .../health/TestOpenContainerHandler.java | 168 +++ .../health/TestQuasiClosedContainerHandler.java | 238 ++++ .../container/replication/health/package-info.java | 11 +- .../scm/metadata/TestSCMMetadataStoreImpl.java | 77 ++ hadoop-hdds/test-utils/pom.xml | 4 +- .../org/apache/ozone/test/GenericTestUtils.java | 42 + hadoop-hdds/tools/pom.xml | 6 +- .../hdds/scm/cli/container/InfoSubcommand.java | 6 +- .../hdds/scm/cli/container/TestInfoSubCommand.java | 22 +- hadoop-ozone/client/pom.xml | 6 +- .../client/checksum/BaseFileChecksumHelper.java | 40 +- .../checksum/ReplicatedFileChecksumHelper.java | 4 +- .../ozone/client/io/BlockOutputStreamEntry.java | 19 +- .../client/io/BlockOutputStreamEntryPool.java | 20 +- .../ozone/client/io/ECBlockOutputStreamEntry.java | 16 +- .../client/io/ECBlockOutputStreamEntryPool.java | 7 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 4 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 28 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 7 +- .../checksum/TestReplicatedFileChecksumHelper.java | 11 +- hadoop-ozone/common/pom.xml | 12 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 39 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 11 + .../ozone/om/protocol/OzoneManagerProtocol.java | 14 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 20 + hadoop-ozone/csi/pom.xml | 6 +- .../dev-support/checks/_mvn_unit_report.sh | 2 +- hadoop-ozone/dev-support/checks/rat.sh | 4 +- .../compose/ozone-topology/docker-compose.yaml | 15 + .../src/main/compose/ozone-topology/docker-config | 5 + .../dist/src/main/compose/ozone-topology/test.sh | 2 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 10 +- hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 12 +- .../src/main/smoketest/admincli/pipeline.robot | 5 +- .../dist/src/main/smoketest/ec/basic.robot | 6 +- .../src/main/smoketest/freon/echoRPCLoad.robot | 42 + .../dist/src/main/smoketest/recon/recon-api.robot | 14 + .../main/smoketest/recon/recon-fso-nssummary.robot | 146 +++ .../dist/src/main/smoketest/s3/commonawslib.robot | 2 + .../dist/src/main/smoketest/s3/freon.robot | 44 + .../dist/src/main/smoketest/s3/objectputget.robot | 6 + hadoop-ozone/dist/src/shell/ozone/ozone | 3 +- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 22 +- .../hadoop/ozone/TestMiniChaosOzoneCluster.java | 28 +- hadoop-ozone/integration-test/pom.xml | 94 +- .../fs/contract/AbstractContractUnbufferTest.java | 160 --- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 80 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 203 ++++ .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 247 ++++ .../hdds/scm/TestSCMDbCheckpointServlet.java | 31 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 32 +- .../metrics/TestSCMContainerManagerMetrics.java | 81 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 42 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 40 +- .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 22 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 33 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 35 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 21 +- .../client/rpc/read/TestChunkInputStream.java | 20 + .../ozone/container/TestECContainerRecovery.java | 84 +- .../TestDataScanner.java} | 31 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 11 +- .../om/{TestOmLDBCli.java => TestLDBCli.java} | 124 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 7 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 39 +- .../hadoop/ozone/scm/TestCloseContainer.java | 26 +- .../hadoop/ozone/scm/TestContainerSmallFile.java | 84 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 47 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 31 +- .../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 30 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 22 +- .../hadoop/ozone/scm/TestXceiverClientManager.java | 103 +- .../hadoop/ozone/scm/TestXceiverClientMetrics.java | 19 +- .../scm/node/TestDecommissionAndMaintenance.java | 210 ++-- .../hadoop/ozone/scm/node/TestQueryNode.java | 29 +- .../scm/pipeline/TestPipelineManagerMXBean.java | 35 +- .../TestSCMPipelineBytesWrittenMetrics.java | 28 +- .../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 69 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 21 +- .../src/test/resources/testSequenceFile | Bin 0 -> 96 bytes .../src/main/proto/OmClientProtocol.proto | 33 +- .../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 45 +- hadoop-ozone/ozone-manager/pom.xml | 16 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 665 +---------- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +- .../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 + .../hadoop/ozone/om/OMPerformanceMetrics.java | 92 ++ .../apache/hadoop/ozone/om/OmMetadataReader.java | 4 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 +- .../apache/hadoop/ozone/om/OzoneConfigUtil.java | 61 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 153 ++- .../java/org/apache/hadoop/ozone/om/ScmClient.java | 81 +- .../om/hashcodegenerator/OMHashCodeGenerator.java | 2 +- .../StringOMHashCodeGeneratorImpl.java | 2 +- .../ozone/om/lock/OBSKeyPathLockStrategy.java | 38 +- .../hadoop/ozone/om/lock/OzoneLockStrategy.java | 15 +- .../ozone/om/lock/RegularBucketLockStrategy.java | 8 +- .../RangerClientMultiTenantAccessController.java | 27 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 13 + .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../hadoop/ozone/om/request/OMEchoRPCRequest.java} | 25 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 11 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 + .../request/s3/security/S3SecretRequestHelper.java | 2 +- .../protocolPB/OzoneManagerRequestHandler.java | 29 + .../ozone/security/acl/OzoneNativeAuthorizer.java | 22 +- .../webapps/ozoneManager/om-overview.html | 20 + .../resources/webapps/ozoneManager/ozoneManager.js | 30 + .../org/apache/hadoop/ozone/om/OmTestManagers.java | 2 +- .../hadoop/ozone/om/TestOzoneConfigUtil.java | 42 + .../org/apache/hadoop/ozone/om/TestScmClient.java | 191 +++ .../om/request/key/TestOMKeyCreateRequest.java | 47 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 5 + .../ozone/om/request/key/TestOMKeyRequest.java | 5 +- .../s3/security/TestS3GetSecretRequest.java | 206 ++-- .../ozone/om/response/TestCleanupTableInfo.java | 3 + .../security/acl/TestOzoneAdministrators.java | 66 +- .../security/acl/TestOzoneNativeAuthorizer.java | 8 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 3 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 3 +- hadoop-ozone/ozonefs-common/pom.xml | 5 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 138 ++- .../fs/ozone/BasicRootedOzoneFileSystem.java | 154 ++- hadoop-ozone/ozonefs-hadoop2/pom.xml | 6 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 6 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 + hadoop-ozone/ozonefs-shaded/pom.xml | 4 + hadoop-ozone/ozonefs/pom.xml | 5 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 + .../hadoop/ozone/recon/ReconControllerModule.java | 4 +- .../hadoop/ozone/recon/ReconRestServletModule.java | 2 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 929 +-------------- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 57 +- .../ozone/recon/api/filters/ReconAdminFilter.java | 12 +- .../recon/api/handlers/BucketEntityHandler.java | 152 +++ .../ozone/recon/api/handlers/BucketHandler.java | 207 ++++ .../recon/api/handlers/DirectoryEntityHandler.java | 150 +++ .../ozone/recon/api/handlers/EntityHandler.java | 342 ++++++ .../ozone/recon/api/handlers/FSOBucketHandler.java | 279 +++++ .../ozone/recon/api/handlers/KeyEntityHandler.java | 93 ++ .../recon/api/handlers/RootEntityHandler.java | 176 +++ .../recon/api/handlers/UnknownEntityHandler.java | 82 ++ .../recon/api/handlers/VolumeEntityHandler.java | 160 +++ .../EntityType.java => handlers/package-info.java} | 13 +- .../ozone/recon/api/types/ContainerKeyPrefix.java | 9 + .../hadoop/ozone/recon/api/types/EntityType.java | 85 +- ...ainerKeyPrefix.java => KeyPrefixContainer.java} | 50 +- .../hadoop/ozone/recon/api/types/NSSummary.java | 7 +- .../recon/recovery/ReconOmMetadataManagerImpl.java | 10 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +- .../scm/ReconStorageContainerManagerFacade.java | 11 +- .../recon/spi/ReconContainerMetadataManager.java | 17 + .../recon/spi/impl/KeyPrefixContainerCodec.java | 92 ++ .../impl/ReconContainerMetadataManagerImpl.java | 304 +++-- .../ozone/recon/spi/impl/ReconDBDefinition.java | 12 +- .../ozone/recon/spi/impl/ReconDBProvider.java | 11 +- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 94 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 206 +--- .../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 217 ++++ .../webapps/recon/ozone-recon-web/api/db.json | 46 +- .../webapps/recon/ozone-recon-web/api/routes.json | 3 +- .../components/autoReloadPanel/autoReloadPanel.tsx | 44 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 8 + .../src/views/datanodes/datanodes.tsx | 29 +- .../src/views/overview/overview.tsx | 35 +- .../src/views/pipelines/pipelines.less | 2 +- .../src/views/pipelines/pipelines.tsx | 33 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 44 +- .../ozone/recon/api/TestNSSummaryEndpoint.java | 663 ----------- .../recon/api/TestNSSummaryEndpointWithFSO.java | 1249 ++++++++++++++++++++ .../ozone/recon/api/filters/TestAdminFilter.java | 23 + .../ozone/recon/fsck/TestContainerHealthTask.java | 3 +- .../recovery/TestReconOmMetadataManagerImpl.java | 11 + .../TestReconContainerMetadataManagerImpl.java | 25 + .../ozone/recon/tasks/TestNSSummaryTask.java | 541 --------- .../recon/tasks/TestNSSummaryTaskWithFSO.java | 604 ++++++++++ hadoop-ozone/s3gateway/pom.xml | 42 - .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 7 + .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 74 +- .../ozone/s3/signature/StringToSignProducer.java | 1 + .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 153 ++- .../ozone/s3/metrics/TestS3GatewayMetrics.java | 10 + .../hadoop/ozone/debug/DBDefinitionFactory.java | 3 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 61 +- .../ozone/debug/container/ContainerCommands.java | 12 + .../ozone/debug/container/ExportSubcommand.java | 35 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +- .../ozone/freon/FreonReplicationOptions.java | 5 +- .../hadoop/ozone/freon/OmRPCLoadGenerator.java | 120 ++ .../hadoop/ozone/freon/S3BucketGenerator.java | 73 ++ .../hadoop/ozone/freon/S3EntityGenerator.java | 65 + .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 47 +- .../freon/containergenerator/GeneratorOm.java | 14 +- .../hadoop/ozone/shell/ReplicationOptions.java | 18 +- .../ozone/shell/ShellReplicationOptions.java | 3 +- .../ozone/shell/bucket/InfoBucketHandler.java | 2 +- .../ozone/shell/bucket/ListBucketHandler.java | 17 +- pom.xml | 44 +- 393 files changed, 14348 insertions(+), 6595 deletions(-) diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml index aba10935e9,4d059d2d60..c10344756a --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@@ -3296,13 -3338,25 +3347,33 @@@ will create intermediate directories. </description> </property> + <property> + <name>ozone.om.snapshot.cache.max.size</name> + <value>10</value> + <tag>OZONE, OM</tag> + <description> + Size of the OM Snapshot LRU cache. This is the maximum number of open OM Snapshot RocksDb instances + that will be held in memory at any time. + </description> + </property> + + <property> + <name>ozone.om.container.location.cache.size</name> + <value>100000</value> + <tag>OZONE, OM</tag> + <description> + The size of the container locations cache in Ozone Manager. This cache allows Ozone Manager to populate + block locations in key-read responses without calling SCM, thus increases Ozone Manager read performance. + </description> + </property> + + <property> + <name>ozone.om.container.location.cache.ttl</name> + <value>360m</value> + <tag>OZONE, OM</tag> + <description> + The time to live for container location cache in Ozone. + </description> + </property> + - </configuration> diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index fd90343ae4,aca30a7b20..5478e8658a --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@@ -90,7 -90,7 +90,8 @@@ public final class DBStoreBuilder private RocksDBConfiguration rocksDBConfiguration; // Flag to indicate if the RocksDB should be opened readonly. private boolean openReadOnly = false; + private int maxFSSnapshots = 0; + private final DBProfile defaultCfProfile; /** * Create DBStoreBuilder from a generic DBDefinition. @@@ -178,27 -175,26 +176,30 @@@ Set<TableConfig> tableConfigs = makeTableConfigs(); - if (rocksDBOption == null) { - rocksDBOption = getDefaultDBOptions(tableConfigs); - } + try { + if (rocksDBOption == null) { + rocksDBOption = getDefaultDBOptions(tableConfigs); + } - ManagedWriteOptions writeOptions = new ManagedWriteOptions(); - writeOptions.setSync(rocksDBConfiguration.getSyncOption()); + ManagedWriteOptions writeOptions = new ManagedWriteOptions(); + writeOptions.setSync(rocksDBConfiguration.getSyncOption()); - File dbFile = getDBFile(); - if (!dbFile.getParentFile().exists()) { - throw new IOException("The DB destination directory should exist."); - } + File dbFile = getDBFile(); + if (!dbFile.getParentFile().exists()) { + throw new IOException("The DB destination directory should exist."); + } - return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs, - registry, openReadOnly, maxFSSnapshots); + return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs, - registry, openReadOnly, dbJmxBeanNameName); ++ registry, openReadOnly, maxFSSnapshots, dbJmxBeanNameName); + } finally { + tableConfigs.forEach(TableConfig::close); + } } + public DBStoreBuilder setMaxFSSnapshots(int maxFSSnapshots) { + this.maxFSSnapshots = maxFSSnapshots; + return this; + } - public DBStoreBuilder setName(String name) { dbname = name; return this; diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java index 6835fe559a,943d58ef1d..f227ab81fd --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java @@@ -95,14 -92,8 +96,18 @@@ public class RDBCheckpointManager imple return null; } + /** + * Create RocksDB snapshot by saving a checkpoint to a directory. + * + * @param parentDir The directory where the checkpoint needs to be created. + * @return RocksDB specific Checkpoint information object. + */ + public RocksDBCheckpoint createCheckpoint(String parentDir) { + return createCheckpoint(parentDir, null); + } + + @Override + public void close() throws IOException { + checkpoint.close(); + } } diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java index 4b98213af1,be8cfb9900..9fabf5e9df --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java @@@ -60,36 -57,29 +60,40 @@@ public class RDBStore implements DBStor private ObjectName statMBeanName; private final RDBCheckpointManager checkPointManager; private final String checkpointsParentDir; + private final String snapshotsParentDir; private final RDBMetrics rdbMetrics; + private final RocksDBCheckpointDiffer rocksDBCheckpointDiffer; + private final String dbJmxBeanName; @VisibleForTesting public RDBStore(File dbFile, ManagedDBOptions options, Set<TableConfig> families) throws IOException { this(dbFile, options, new ManagedWriteOptions(), families, - new CodecRegistry(), false, 1000); - new CodecRegistry(), false, null); ++ new CodecRegistry(), false, 1000, null); } ++ @SuppressWarnings("parameternumber") public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedWriteOptions writeOptions, Set<TableConfig> families, - CodecRegistry registry, boolean readOnly, int maxFSSnapshots) - throws IOException { - CodecRegistry registry, boolean readOnly, ++ CodecRegistry registry, boolean readOnly, int maxFSSnapshots, + String dbJmxBeanNameName) throws IOException { Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); Preconditions.checkNotNull(families); Preconditions.checkArgument(!families.isEmpty()); codecRegistry = registry; dbLocation = dbFile; + dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() : + dbJmxBeanNameName; try { + rocksDBCheckpointDiffer = + new RocksDBCheckpointDiffer( + dbLocation.getAbsolutePath(), maxFSSnapshots, + Paths.get(dbLocation.getParent(), "db.checkpoints").toString(), + Paths.get(dbLocation.getParent(), "db.savedSSTFiles").toString(), + dbLocation.getAbsolutePath(), 0, "Snapshot_"); + rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions); + db = RocksDatabase.open(dbFile, dbOptions, writeOptions, families, readOnly); diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index caa61e8d69,65a2f67883..ab7bacb05b --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@@ -814,8 -800,37 +819,42 @@@ public final class OmUtils return printString.toString(); } + // Key points to entire bucket's snapshot + public static boolean isBucketSnapshotIndicator(String key) { + return key.startsWith(OM_SNAPSHOT_INDICATOR) && key.split("/").length == 2; + } ++ + public static String format(List<ServiceInfo> nodes, int port, + String leaderId) { + StringBuilder sb = new StringBuilder(); + // Ensuring OM's are printed in correct order + List<ServiceInfo> omNodes = nodes.stream() + .filter(node -> node.getNodeType() == HddsProtos.NodeType.OM) + .sorted(Comparator.comparing(ServiceInfo::getHostname)) + .collect(Collectors.toList()); + int count = 0; + for (ServiceInfo info : omNodes) { + // Printing only the OM's running + if (info.getNodeType() == HddsProtos.NodeType.OM) { + String role = + info.getOmRoleInfo().getNodeId().equals(leaderId) ? "LEADER" : + "FOLLOWER"; + sb.append( + String.format( + " { HostName: %s | Node-Id: %s | Ratis-Port : %d | Role: %s} ", + info.getHostname(), + info.getOmRoleInfo().getNodeId(), + port, + role + )); + count++; + } + } + // Print Stand-alone if only one OM exists + if (count == 1) { + return "STANDALONE"; + } else { + return sb.toString(); + } + } } diff --cc hadoop-ozone/dist/src/main/license/jar-report.txt index 9581b6dbfb,ac5a3ab9a1..ba25a09545 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@@ -255,7 -245,7 +250,8 @@@ share/ozone/lib/ratis-thirdparty-misc.j share/ozone/lib/ratis-tools.jar share/ozone/lib/re2j.jar share/ozone/lib/reflections.jar +share/ozone/lib/rocksdb-checkpoint-differ.jar + share/ozone/lib/reload4j.jar share/ozone/lib/rocksdbjni.jar share/ozone/lib/simpleclient_common.jar share/ozone/lib/simpleclient_dropwizard.jar diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d46dda9ff3,88501120bf..5c9b3409a5 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -122,7 -122,7 +122,10 @@@ enum Type SetRangerServiceVersion = 107; RangerBGSync = 109; - CreateSnapshot = 110; ++ + EchoRPC = 110; ++ ++ CreateSnapshot = 111; } message OMRequest { @@@ -227,7 -227,7 +230,11 @@@ optional SetRangerServiceVersionRequest SetRangerServiceVersionRequest = 107; optional RangerBGSyncRequest RangerBGSyncRequest = 109; - optional CreateSnapshotRequest CreateSnapshotRequest = 110; ++ + optional EchoRPCRequest EchoRPCRequest = 110; ++ ++ optional CreateSnapshotRequest CreateSnapshotRequest = 111; ++ } message OMResponse { @@@ -325,7 -325,7 +332,11 @@@ optional SetRangerServiceVersionResponse SetRangerServiceVersionResponse = 107; optional RangerBGSyncResponse RangerBGSyncResponse = 109; - optional CreateSnapshotResponse CreateSnapshotResponse = 110; ++ + optional EchoRPCResponse EchoRPCResponse = 110; ++ ++ optional CreateSnapshotResponse CreateSnapshotResponse = 111; ++ } enum Status { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index d901336223,4ea8c52bf4..a2de68dcc2 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@@ -171,17 -161,37 +159,17 @@@ public class KeyManagerImpl implements private BackgroundService openKeyCleanupService; - @VisibleForTesting - public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, - OMMetadataManager metadataManager, OzoneConfiguration conf, - OzoneBlockTokenSecretManager secretManager, - OMPerformanceMetrics metrics) { - this(null, new ScmClient(scmBlockClient, null, conf), metadataManager, - conf, secretManager, null, metrics); - } - - @VisibleForTesting - public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, - StorageContainerLocationProtocol scmContainerClient, - OMMetadataManager metadataManager, OzoneConfiguration conf, String omId, - OzoneBlockTokenSecretManager secretManager, - OMPerformanceMetrics metrics) { - this(null, new ScmClient(scmBlockClient, scmContainerClient, conf), - metadataManager, conf, secretManager, null, - metrics); - } - public KeyManagerImpl(OzoneManager om, ScmClient scmClient, - OzoneConfiguration conf, String omId) { - this (om, scmClient, om.getMetadataManager(), conf, omId, - om.getBlockTokenMgr(), om.getKmsProvider(), om.getPrefixManager()); + OzoneConfiguration conf, OMPerformanceMetrics metrics) { + this (om, scmClient, om.getMetadataManager(), conf, + om.getBlockTokenMgr(), om.getKmsProvider(), metrics); } - @SuppressWarnings("parameternumber") public KeyManagerImpl(OzoneManager om, ScmClient scmClient, - OMMetadataManager metadataManager, OzoneConfiguration conf, String omId, + OMMetadataManager metadataManager, OzoneConfiguration conf, OzoneBlockTokenSecretManager secretManager, - KeyProviderCryptoExtension kmsProvider, PrefixManager prefixManager) { + KeyProviderCryptoExtension kmsProvider, + OMPerformanceMetrics metrics) { this.scmBlockSize = (long) conf .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES); diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index d3a14e267f,0000000000..42771f66f9 mode 100644,000000..100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@@ -1,499 -1,0 +1,501 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.om; + +import java.io.IOException; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; ++import org.apache.hadoop.hdds.server.OzoneAdmins; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditAction; +import org.apache.hadoop.ozone.audit.AuditEventStatus; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.audit.Auditor; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; +import org.apache.hadoop.ozone.security.acl.RequestContext; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.slf4j.Logger; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS; +import static org.apache.hadoop.ozone.om.KeyManagerImpl.getRemoteUser; +import static org.apache.hadoop.ozone.om.OzoneManager.getS3Auth; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; +import org.apache.hadoop.ozone.security.acl.OzoneAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType; +import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; + +/** + * OM Metadata Reading class for the OM and Snapshot managers. + * + * This abstraction manages all the metadata key/acl reading + * from a rocksDb instance, for both the OM and OM snapshots. + */ +public class OmMetadataReader implements IOmMetadataReader, Auditor { + private final KeyManager keyManager; + private final PrefixManager prefixManager; + private final VolumeManager volumeManager; + private final BucketManager bucketManager; + private final OzoneManager ozoneManager; + private final boolean isAclEnabled; + private final IAccessAuthorizer accessAuthorizer; + private final boolean isNativeAuthorizerEnabled; + private final OmMetadataReaderMetrics metrics; + private final Logger log; + private final AuditLogger audit; + + public OmMetadataReader(KeyManager keyManager, + PrefixManager prefixManager, + OzoneManager ozoneManager, + Logger log, + AuditLogger audit, + OmMetadataReaderMetrics omMetadataReaderMetrics) { + this.keyManager = keyManager; + this.bucketManager = ozoneManager.getBucketManager(); + this.volumeManager = ozoneManager.getVolumeManager(); + this.prefixManager = prefixManager; + OzoneConfiguration configuration = ozoneManager.getConfiguration(); + this.ozoneManager = ozoneManager; + this.isAclEnabled = ozoneManager.getAclsEnabled(); + this.log = log; + this.audit = audit; + boolean allowListAllVolumes = ozoneManager.getAllowListAllVolumes(); + metrics = omMetadataReaderMetrics; + if (isAclEnabled) { + accessAuthorizer = getACLAuthorizerInstance(configuration); + if (accessAuthorizer instanceof OzoneNativeAuthorizer) { + OzoneNativeAuthorizer authorizer = + (OzoneNativeAuthorizer) accessAuthorizer; + isNativeAuthorizerEnabled = true; + authorizer.setVolumeManager(volumeManager); + authorizer.setBucketManager(bucketManager); + authorizer.setKeyManager(keyManager); + authorizer.setPrefixManager(prefixManager); - authorizer.setOzoneAdmins(ozoneManager.getOmAdminUsernames()); ++ authorizer.setOzoneAdmins( ++ new OzoneAdmins(ozoneManager.getOmAdminUsernames())); + authorizer.setAllowListAllVolumes(allowListAllVolumes); + } else { + isNativeAuthorizerEnabled = false; + } + } else { + accessAuthorizer = null; + isNativeAuthorizerEnabled = false; + } + } + + /** + * Lookup a key. + * + * @param args - attributes of the key. + * @return OmKeyInfo - the info about the requested key. + * @throws IOException + */ + @Override + public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + if (isAclEnabled) { + checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()); + } + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + metrics.incNumKeyLookups(); + return keyManager.lookupKey(args, getClientAddress()); + } catch (Exception ex) { + metrics.incNumKeyLookupFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY, + auditMap)); + } + } + } + + @Override + public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, boolean allowPartialPrefixes) + throws IOException { + + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + if (isAclEnabled) { + checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()); + } + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + metrics.incNumListStatus(); + return keyManager.listStatus(args, recursive, startKey, numEntries, + getClientAddress(), allowPartialPrefixes); + } catch (Exception ex) { + metrics.incNumListStatusFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess( + OMAction.LIST_STATUS, auditMap)); + } + } + } + + @Override + public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + metrics.incNumGetFileStatus(); + return keyManager.getFileStatus(args, getClientAddress()); + } catch (IOException ex) { + metrics.incNumGetFileStatusFails(); + auditSuccess = false; + audit.logReadFailure( + buildAuditMessageForFailure(OMAction.GET_FILE_STATUS, auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess( + buildAuditMessageForSuccess(OMAction.GET_FILE_STATUS, auditMap)); + } + } + } + + @Override + public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException { + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + if (isAclEnabled) { + checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()); + } + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + metrics.incNumLookupFile(); + return keyManager.lookupFile(args, getClientAddress()); + } catch (Exception ex) { + metrics.incNumLookupFileFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LOOKUP_FILE, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess( + OMAction.LOOKUP_FILE, auditMap)); + } + } + } + + @Override + public List<OmKeyInfo> listKeys(String volumeName, String bucketName, + String startKey, String keyPrefix, int maxKeys) throws IOException { + + ResolvedBucket bucket = ozoneManager.resolveBucketLink( + Pair.of(volumeName, bucketName)); + + if (isAclEnabled) { + checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, + bucket.realVolume(), bucket.realBucket(), keyPrefix); + } + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(); + auditMap.put(OzoneConsts.START_KEY, startKey); + auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys)); + auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix); + + try { + metrics.incNumKeyLists(); + return keyManager.listKeys(bucket.realVolume(), bucket.realBucket(), + startKey, keyPrefix, maxKeys); + } catch (IOException ex) { + metrics.incNumKeyListFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_KEYS, + auditMap)); + } + } + } + + /** + * Returns list of ACLs for given Ozone object. + * + * @param obj Ozone object. + * @throws IOException if there is error. + */ + public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException { + boolean auditSuccess = true; + + try { + if (isAclEnabled) { + checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.READ_ACL, + obj.getVolumeName(), obj.getBucketName(), obj.getKeyName()); + } + metrics.incNumGetAcl(); + switch (obj.getResourceType()) { + case VOLUME: + return volumeManager.getAcl(obj); + case BUCKET: + return bucketManager.getAcl(obj); + case KEY: + return keyManager.getAcl(obj); + case PREFIX: + return prefixManager.getAcl(obj); + + default: + throw new OMException("Unexpected resource type: " + + obj.getResourceType(), INVALID_REQUEST); + } + } catch (Exception ex) { + auditSuccess = false; + audit.logReadFailure( + buildAuditMessageForFailure(OMAction.GET_ACL, obj.toAuditMap(), ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess( + buildAuditMessageForSuccess(OMAction.GET_ACL, obj.toAuditMap())); + } + } + } + + /** + * Checks if current caller has acl permissions. + * + * @param resType - Type of ozone resource. Ex volume, bucket. + * @param store - Store type. i.e Ozone, S3. + * @param acl - type of access to be checked. + * @param vol - name of volume + * @param bucket - bucket name + * @param key - key + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied. + */ + void checkAcls(ResourceType resType, StoreType store, + ACLType acl, String vol, String bucket, String key) + throws IOException { + UserGroupInformation user; + if (getS3Auth() != null) { + String principal = + OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId()); + user = UserGroupInformation.createRemoteUser(principal); + } else { + user = ProtobufRpcEngine.Server.getRemoteUser(); + } + + InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp(); + String volumeOwner = ozoneManager.getVolumeOwner(vol, acl, resType); + String bucketOwner = ozoneManager.getBucketOwner(vol, bucket, acl, resType); + + OzoneAclUtils.checkAllAcls(this, resType, store, acl, + vol, bucket, key, volumeOwner, bucketOwner, + user != null ? user : getRemoteUser(), + remoteIp != null ? remoteIp : + ozoneManager.getOmRpcServerAddr().getAddress(), + remoteIp != null ? remoteIp.getHostName() : + ozoneManager.getOmRpcServerAddr().getHostName()); + } + + + /** + * CheckAcls for the ozone object. + * + * @return true if permission granted, false if permission denied. + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied + * and throwOnPermissionDenied set to true. + */ + @SuppressWarnings("parameternumber") + public boolean checkAcls(ResourceType resType, StoreType storeType, + ACLType aclType, String vol, String bucket, String key, + UserGroupInformation ugi, InetAddress remoteAddress, String hostName, + boolean throwIfPermissionDenied, String owner) + throws OMException { + OzoneObj obj = OzoneObjInfo.Builder.newBuilder() + .setResType(resType) + .setStoreType(storeType) + .setVolumeName(vol) + .setBucketName(bucket) + .setKeyName(key).build(); + RequestContext context = RequestContext.newBuilder() + .setClientUgi(ugi) + .setIp(remoteAddress) + .setHost(hostName) + .setAclType(ACLIdentityType.USER) + .setAclRights(aclType) + .setOwnerName(owner) + .build(); + + return checkAcls(obj, context, throwIfPermissionDenied); + } + + /** + * CheckAcls for the ozone object. + * + * @return true if permission granted, false if permission denied. + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied + * and throwOnPermissionDenied set to true. + */ + public boolean checkAcls(OzoneObj obj, RequestContext context, + boolean throwIfPermissionDenied) + throws OMException { + + if (!accessAuthorizer.checkAccess(obj, context)) { + if (throwIfPermissionDenied) { + String volumeName = obj.getVolumeName() != null ? + "Volume:" + obj.getVolumeName() + " " : ""; + String bucketName = obj.getBucketName() != null ? + "Bucket:" + obj.getBucketName() + " " : ""; + String keyName = obj.getKeyName() != null ? + "Key:" + obj.getKeyName() : ""; + log.warn("User {} doesn't have {} permission to access {} {}{}{}", + context.getClientUgi().getUserName(), context.getAclRights(), + obj.getResourceType(), volumeName, bucketName, keyName); + throw new OMException("User " + context.getClientUgi().getUserName() + + " doesn't have " + context.getAclRights() + + " permission to access " + obj.getResourceType() + " " + + volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED); + } + return false; + } else { + return true; + } + } + + /** + * Returns an instance of {@link IAccessAuthorizer}. + * Looks up the configuration to see if there is custom class specified. + * Constructs the instance by passing the configuration directly to the + * constructor to achieve thread safety using final fields. + * + * @param conf + * @return IAccessAuthorizer + */ + private IAccessAuthorizer getACLAuthorizerInstance(OzoneConfiguration conf) { + Class<? extends IAccessAuthorizer> clazz = conf.getClass( + OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizer.class, + IAccessAuthorizer.class); + return ReflectionUtils.newInstance(clazz, conf); + } + + private static String getClientAddress() { + String clientMachine = Server.getRemoteAddress(); + if (clientMachine == null) { //not a RPC client + clientMachine = ""; + } + return clientMachine; + } + + @Override + public AuditMessage buildAuditMessageForSuccess(AuditAction op, + Map<String, String> auditMap) { + + return new AuditMessage.Builder() + .setUser(getRemoteUserName()) + .atIp(Server.getRemoteAddress()) + .forOperation(op) + .withParams(auditMap) + .withResult(AuditEventStatus.SUCCESS) + .build(); + } + + @Override + public AuditMessage buildAuditMessageForFailure(AuditAction op, + Map<String, String> auditMap, Throwable throwable) { + + return new AuditMessage.Builder() + .setUser(getRemoteUserName()) + .atIp(Server.getRemoteAddress()) + .forOperation(op) + .withParams(auditMap) + .withResult(AuditEventStatus.FAILURE) + .withException(throwable) + .build(); + } + + /** + * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise. + * + * @return if native authorizer is enabled. + */ + public boolean isNativeAuthorizerEnabled() { + return isNativeAuthorizerEnabled; + } + + public IAccessAuthorizer getAccessAuthorizer() { + return accessAuthorizer; + } + + private ResourceType getResourceType(OmKeyArgs args) { + if (args.getKeyName() == null || args.getKeyName().length() == 0) { + return ResourceType.BUCKET; + } + return ResourceType.KEY; + } + + +} diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index dcbed30bd4,0000000000..ffdd136877 mode 100644,000000..100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@@ -1,199 -1,0 +1,199 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import org.apache.hadoop.hdds.utils.db.DBCheckpoint; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; + +import java.io.IOException; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.util.concurrent.ExecutionException; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; + + +/** + * This class is used to manage/create OM snapshots. + */ +public final class OmSnapshotManager { + private final OzoneManager ozoneManager; + private final LoadingCache<String, OmSnapshot> snapshotCache; + + private static final Logger LOG = + LoggerFactory.getLogger(OmSnapshotManager.class); + + OmSnapshotManager(OzoneManager ozoneManager) { + this.ozoneManager = ozoneManager; + + // size of lru cache + int cacheSize = ozoneManager.getConfiguration().getInt( + OzoneConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE, + OzoneConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT); + + CacheLoader<String, OmSnapshot> loader; + loader = new CacheLoader<String, OmSnapshot>() { + @Override + + // load the snapshot into the cache if not already there + @Nonnull + public OmSnapshot load(@Nonnull String snapshotTableKey) + throws IOException { + SnapshotInfo snapshotInfo; + // see if the snapshot exists + snapshotInfo = getSnapshotInfo(snapshotTableKey); + + // read in the snapshot + OzoneConfiguration conf = ozoneManager.getConfiguration(); + OMMetadataManager snapshotMetadataManager; + + // Create the snapshot metadata manager by finding the corresponding + // RocksDB instance, creating an OmMetadataManagerImpl instance based on + // that + try { + snapshotMetadataManager = OmMetadataManagerImpl + .createSnapshotMetadataManager( + conf, snapshotInfo.getCheckpointDirName()); + } catch (IOException e) { + LOG.error("Failed to retrieve snapshot: {}, {}", snapshotTableKey, e); + throw e; + } + + // create the other manager instances based on snapshot metadataManager + PrefixManagerImpl pm = new PrefixManagerImpl(snapshotMetadataManager, + false); + KeyManagerImpl km = new KeyManagerImpl(null, - ozoneManager.getScmClient(), snapshotMetadataManager, conf, null, ++ ozoneManager.getScmClient(), snapshotMetadataManager, conf, + ozoneManager.getBlockTokenSecretManager(), - ozoneManager.getKmsProvider(), pm); ++ ozoneManager.getKmsProvider(), ozoneManager.getPerfMetrics()); + + return new OmSnapshot(km, pm, ozoneManager, + snapshotInfo.getVolumeName(), + snapshotInfo.getBucketName(), + snapshotInfo.getName()); + } + }; + + RemovalListener<String, OmSnapshot> removalListener + = notification -> { + try { + // close snapshot's rocksdb on eviction + notification.getValue().close(); + } catch (IOException e) { + LOG.error("Failed to close snapshot: {} {}", + notification.getKey(), e); + } + }; + // init LRU cache + snapshotCache = CacheBuilder.newBuilder() + .maximumSize(cacheSize) + .removalListener(removalListener) + .build(loader); + } + + /** + * Creates snapshot checkpoint that corresponds to snapshotInfo. + * @param omMetadataManager the metadata manager + * @param snapshotInfo The metadata of snapshot to be created + * @return instance of DBCheckpoint + */ + public static DBCheckpoint createOmSnapshotCheckpoint( + OMMetadataManager omMetadataManager, SnapshotInfo snapshotInfo) + throws IOException { + RDBStore store = (RDBStore) omMetadataManager.getStore(); + return store.getSnapshot(snapshotInfo.getCheckpointDirName()); + } + + // Get OmSnapshot if the keyname has ".snapshot" key indicator + public IOmMetadataReader checkForSnapshot(String volumeName, + String bucketName, String keyname) + throws IOException { + if (keyname == null) { + return ozoneManager.getOmMetadataReader(); + } + + // see if key is for a snapshot + String[] keyParts = keyname.split("/"); + if (isSnapshotKey(keyParts)) { + String snapshotName = keyParts[1]; + if (snapshotName == null || snapshotName.isEmpty()) { + // don't allow snapshot indicator without snapshot name + throw new OMException(INVALID_KEY_NAME); + } + String snapshotTableKey = SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName); + + // retrieve the snapshot from the cache + try { + return snapshotCache.get(snapshotTableKey); + } catch (ExecutionException e) { + throw new IOException(e.getCause()); + } + } else { + return ozoneManager.getOmMetadataReader(); + } + } + + public SnapshotInfo getSnapshotInfo(String volumeName, + String bucketName, String snapshotName) + throws IOException { + return getSnapshotInfo(SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName)); + } + + private SnapshotInfo getSnapshotInfo(String key) throws IOException { + SnapshotInfo snapshotInfo; + try { + snapshotInfo = ozoneManager.getMetadataManager() + .getSnapshotInfoTable() + .get(key); + } catch (IOException e) { + LOG.error("Snapshot {}: not found: {}", key, e); + throw e; + } + if (snapshotInfo == null) { + throw new OMException(KEY_NOT_FOUND); + } + return snapshotInfo; + } + + public static String getSnapshotPrefix(String snapshotName) { + return OM_SNAPSHOT_INDICATOR + OM_KEY_PREFIX + + snapshotName + OM_KEY_PREFIX; + } + + public static boolean isSnapshotKey(String[] keyParts) { + return (keyParts.length > 1) && + (keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0); + } +} diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9664e602f4,8d6ae64ddf..8013e1bd15 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -215,10 -222,9 +217,8 @@@ import static org.apache.hadoop.ozone.O import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED_DEFAULT; - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; - import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX; @@@ -565,10 -574,18 +569,19 @@@ public final class OzoneManager extend this.isS3MultiTenancyEnabled = OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf); - // Get admin list - omAdminUsernames = getOzoneAdminsFromConfig(configuration); - + metrics = OMMetrics.create(); + // Get admin list + Collection<String> omAdminUsernames = + OzoneConfigUtil.getOzoneAdminsFromConfig(configuration); + Collection<String> omAdminGroups = + OzoneConfigUtil.getOzoneAdminsGroupsFromConfig(configuration); + omAdmins = new OzoneAdmins(omAdminUsernames, omAdminGroups); + + Collection<String> s3AdminUsernames = + OzoneConfigUtil.getS3AdminsFromConfig(configuration); + Collection<String> s3AdminGroups = + OzoneConfigUtil.getS3AdminsGroupsFromConfig(configuration); + s3OzoneAdmins = new OzoneAdmins(s3AdminUsernames, s3AdminGroups); instantiateServices(false); // Create special volume s3v which is required for S3G. @@@ -704,11 -722,9 +717,12 @@@ } prefixManager = new PrefixManagerImpl(metadataManager, isRatisEnabled); + perfMetrics = OMPerformanceMetrics.register(); keyManager = new KeyManagerImpl(this, scmClient, configuration, - omStorage.getOmId()); - perfMetrics); ++ perfMetrics); + omMetadataReader = new OmMetadataReader(keyManager, prefixManager, + this, LOG, AUDIT, metrics); + omSnapshotManager = new OmSnapshotManager(this); if (withNewSnapshot) { Integer layoutVersionInDB = getLayoutVersionInDB(); @@@ -1423,6 -1464,6 +1437,10 @@@ return metrics; } ++ public OMPerformanceMetrics getPerfMetrics() { ++ return perfMetrics; ++ } ++ /** * Start service. */ @@@ -3805,24 -4109,22 +3834,13 @@@ * @param callerUgi Caller UserGroupInformation */ public boolean isAdmin(UserGroupInformation callerUgi) { - if (callerUgi == null) { - return false; - } else { - return isAdmin(callerUgi.getShortUserName()) - || isAdmin(callerUgi.getUserName()); - } + return callerUgi != null && omAdmins.isAdmin(callerUgi); } - @VisibleForTesting - private boolean isAdmin(String username) { - if (omAdminUsernames == null) { - return false; - } else { - return omAdminUsernames.contains(OZONE_ADMINISTRATORS_WILDCARD) || - omAdminUsernames.contains(username); - } + public boolean isS3Admin(UserGroupInformation callerUgi) { + return callerUgi != null && s3OzoneAdmins.isAdmin(callerUgi); } - /** - * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise. - * - * @return if native authorizer is enabled. - */ - public boolean isNativeAuthorizerEnabled() { - return isNativeAuthorizerEnabled; - } - @VisibleForTesting public boolean isRunning() { return omState == State.RUNNING; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
