This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-4944 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c826ec253297e24d8eddafeead71cc987ca32018 Merge: ab84d9e cd8287b Author: Siyao Meng <[email protected]> AuthorDate: Wed Jul 28 15:14:30 2021 -0700 Merge remote-tracking branch 'asf/master' into HDDS-4944 Conflicts: hadoop-ozone/common/pom.xml hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java Change-Id: Iedf0713cfbba4ea95c27f98f74b6c6806b465acd .github/workflows/post-commit.yml | 2 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 72 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 126 +++ .../storage/TestBlockOutputStreamCorrectness.java | 2 +- hadoop-hdds/common/pom.xml | 12 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 9 + .../hadoop/hdds/protocol/DatanodeDetails.java | 9 + .../apache/hadoop/hdds/scm/PlacementPolicy.java | 7 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 17 + .../apache/hadoop/hdds/scm/client/ScmClient.java | 7 + .../protocol/StorageContainerLocationProtocol.java | 8 + .../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 76 ++ .../hdds/upgrade/HDDSLayoutVersionManager.java | 97 ++ .../hadoop/hdds/upgrade/HDDSUpgradeAction.java | 15 +- .../apache/hadoop/hdds/upgrade}/package-info.java | 5 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 5 +- .../org/apache/hadoop/hdds/utils/VersionInfo.java | 10 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 1 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +- .../apache/hadoop/ozone/common/MonotonicClock.java | 81 ++ .../org/apache/hadoop/ozone/common/Storage.java | 41 +- .../apache/hadoop/ozone/common/StorageInfo.java | 33 +- .../hadoop/ozone/conf/OzoneServiceConfig.java | 76 ++ .../apache/hadoop/ozone/conf}/package-info.java | 12 +- .../upgrade/AbstractLayoutVersionManager.java | 162 +++ .../ozone/upgrade/BasicUpgradeFinalizer.java | 367 +++++++ .../DefaultUpgradeFinalizationExecutor.java | 66 ++ .../apache/hadoop/ozone/upgrade/LayoutFeature.java | 79 ++ .../upgrade/LayoutVersionInstanceFactory.java | 253 +++++ .../hadoop/ozone/upgrade/LayoutVersionManager.java | 76 ++ .../ozone/upgrade/LayoutVersionManagerMXBean.java | 18 +- .../hadoop/ozone/upgrade/UpgradeActionHdds.java | 39 +- .../hadoop/ozone/upgrade/UpgradeException.java | 166 +-- .../hadoop/ozone/upgrade/UpgradeFinalizer.java | 195 ++++ .../hadoop/ozone/upgrade/VersionFactoryKey.java | 70 ++ .../apache/hadoop/ozone/upgrade}/package-info.java | 12 +- .../hadoop/ozone/util/ShutdownHookManager.java | 396 ++++++++ .../apache/hadoop/ozone/util}/package-info.java | 12 +- .../main/resources/hdds-version-info.properties | 3 +- .../common/src/main/resources/ozone-default.xml | 9 + .../hdds/upgrade/TestHDDSLayoutVersionManager.java | 102 ++ .../hadoop/hdds/upgrade/test/MockComponent.java | 67 ++ .../ozone/container/ContainerTestHelper.java | 75 ++ .../InjectedUpgradeFinalizationExecutor.java | 128 +++ .../upgrade/TestAbstractLayoutVersionManager.java | 203 ++++ .../ozone/upgrade/TestBasicUpgradeFinalizer.java | 126 +++ .../TestDefaultUpgradeFinalizationExecutor.java | 99 ++ .../upgrade/TestLayoutVersionInstanceFactory.java | 198 ++++ .../ozone/upgrade/TestUpgradeFinalizerActions.java | 210 ++++ .../hadoop/ozone/upgrade/TestUpgradeUtils.java | 54 + .../hadoop/hdds/conf/ConfigFileGenerator.java | 17 +- .../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +- .../org/apache/hadoop/hdds/conf/StorageUnit.java | 8 +- .../hadoop/hdds/freon/FakeRatisFollower.java | 7 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 21 +- .../container/common/DatanodeLayoutStorage.java | 98 ++ ...utVersion.java => HDDSVolumeLayoutVersion.java} | 16 +- .../container/common/helpers/ContainerUtils.java | 5 +- .../container/common/report/ReportManager.java | 3 +- .../common/statemachine/DatanodeConfiguration.java | 4 +- .../common/statemachine/DatanodeStateMachine.java | 53 +- .../common/statemachine/StateContext.java | 3 +- .../commandhandler/DeleteBlocksCommandHandler.java | 2 +- .../FinalizeNewLayoutVersionCommandHandler.java | 121 +++ .../states/endpoint/HeartbeatEndpointTask.java | 65 +- .../states/endpoint/RegisterEndpointTask.java | 60 +- .../common/transport/server/XceiverServerGrpc.java | 3 +- .../server/ratis/ContainerStateMachine.java | 4 +- .../transport/server/ratis/XceiverServerRatis.java | 2 +- .../container/common/utils/HddsVolumeUtil.java | 6 +- .../common/utils/db/DatanodeDBProfile.java | 145 +++ .../container/common/utils/db}/package-info.java | 10 +- .../ozone/container/common/volume/HddsVolume.java | 7 +- .../common/volume/ImmutableVolumeSet.java | 16 + .../container/common/volume/MutableVolumeSet.java | 25 +- .../common/volume/StorageVolumeChecker.java | 54 +- .../ozone/container/common/volume/VolumeSet.java | 3 + .../container/keyvalue/KeyValueContainer.java | 4 +- .../container/metadata/AbstractDatanodeStore.java | 47 +- .../ozone/container/ozoneimpl/ContainerReader.java | 11 +- .../replication/GrpcReplicationClient.java | 3 +- .../container/replication/ReplicationServer.java | 1 + .../replication/ReplicationSupervisor.java | 33 +- .../ozone/container/stream/StreamingClient.java | 1 + .../ozone/container/stream/StreamingServer.java | 1 + .../upgrade/DataNodeUpgradeFinalizer.java | 95 ++ .../upgrade/DatanodeMetadataFeatures.java | 53 + .../upgrade/DatanodeSchemaV2FinalizeAction.java | 48 + .../ScmHAFirstUpgradeLayoutChangeAction.java | 45 + .../UpgradeUtils.java} | 45 +- .../ozone/container/upgrade}/package-info.java | 13 +- .../protocol/StorageContainerDatanodeProtocol.java | 6 +- .../protocol/StorageContainerNodeProtocol.java | 12 +- .../commands/FinalizeNewLayoutVersionCommand.java | 80 ++ ...inerDatanodeProtocolClientSideTranslatorPB.java | 11 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 16 +- .../hadoop/ozone/container/common/ScmTestMock.java | 11 +- .../common/TestDatanodeLayOutVersion.java | 10 +- .../common/helpers/TestContainerUtils.java | 60 ++ .../common/helpers/TestDatanodeVersionFile.java | 4 +- .../states/endpoint/TestHeartbeatEndpointTask.java | 12 +- .../common/volume/TestPeriodicVolumeChecker.java | 131 +++ .../container/keyvalue/TestKeyValueContainer.java | 77 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 78 +- .../replication/TestReplicationSupervisor.java | 4 +- .../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 96 ++ .../docs/content/design/nonrolling-upgrade.md | 32 + hadoop-hdds/docs/content/design/omprepare.md | 60 ++ .../docs/content/design/upgrade-dev-primer.md | 111 ++ .../feature/how-to-do-a-nonrolling-upgrade.md | 54 + .../java/org/apache/hadoop/hdds/NodeDetails.java | 44 +- ...inerLocationProtocolClientSideTranslatorPB.java | 54 +- .../scm/update/client/CRLClientUpdateHandler.java | 5 +- .../update/client/SCMUpdateServiceGrpcClient.java | 3 +- .../hadoop/hdds/server/http/HttpServer2.java | 1 + .../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 35 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 + hadoop-hdds/interface-admin/pom.xml | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 21 + .../dev-support/findbugsExcludeFile.xml | 3 + hadoop-hdds/interface-client/pom.xml | 38 +- .../interface-client/src/main/proto/hdds.proto | 22 +- hadoop-hdds/interface-server/pom.xml | 2 +- .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 23 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 73 +- .../hdds/scm/block/SCMBlockDeletingService.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 549 ++++++++-- .../scm/container/balancer/ContainerBalancer.java | 4 +- .../algorithms/SCMContainerPlacementCapacity.java | 10 +- .../algorithms/SCMContainerPlacementRackAware.java | 102 +- .../algorithms/SCMContainerPlacementRandom.java | 10 +- .../apache/hadoop/hdds/scm/events/SCMEvents.java | 14 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 1 + .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 3 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 3 +- .../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 + .../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 43 +- ...andler.java => HealthyReadOnlyNodeHandler.java} | 41 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 39 + .../hadoop/hdds/scm/node/NodeStateManager.java | 216 +++- .../apache/hadoop/hdds/scm/node/NodeStatus.java | 14 +- ...va => ReadOnlyHealthyToHealthyNodeHandler.java} | 15 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 170 +++- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +- .../scm/pipeline/BackgroundPipelineCreator.java | 15 + .../hadoop/hdds/scm/pipeline/PipelineManager.java | 12 +- .../hdds/scm/pipeline/PipelineManagerV2Impl.java | 25 + .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 49 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 31 +- .../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 4 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 17 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 29 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 63 ++ .../hdds/scm/server/SCMClientProtocolServer.java | 38 +- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 31 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 24 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 1 + .../hadoop/hdds/scm/server/SCMStorageConfig.java | 9 +- .../hdds/scm/server/StorageContainerManager.java | 113 ++- .../scm/server/StorageContainerManagerStarter.java | 12 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 163 +++ .../ScmHAUnfinalizedStateValidationAction.java | 72 ++ .../ScmOnFinalizeActionForDatanodeSchemaV2.java | 46 + .../hdds/scm/server/upgrade}/package-info.java | 5 +- .../update/server/SCMUpdateServiceGrpcServer.java | 3 +- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 19 + .../hadoop/hdds/scm/container/MockNodeManager.java | 26 +- .../hdds/scm/container/SimpleMockNodeManager.java | 40 +- .../TestIncrementalContainerReportHandler.java | 14 +- .../hdds/scm/container/TestReplicationManager.java | 409 ++++++-- .../algorithms/TestContainerPlacementFactory.java | 41 +- .../TestSCMContainerPlacementCapacity.java | 6 +- .../TestSCMContainerPlacementRackAware.java | 112 +- .../TestSCMContainerPlacementRandom.java | 6 +- .../hdds/scm/node/TestContainerPlacement.java | 23 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 2 +- .../hdds/scm/node/TestNodeReportHandler.java | 16 +- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 57 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 532 +++++++++- .../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 17 +- .../hdds/scm/node/states/TestNodeStateMap.java | 26 +- .../hdds/scm/pipeline/MockPipelineManager.java | 10 + .../scm/pipeline/TestPipelinePlacementPolicy.java | 71 +- .../scm/pipeline/TestRatisPipelineProvider.java | 44 +- .../TestSCMHAUnfinalizedStateValidationAction.java | 128 +++ .../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++ .../hadoop/hdds/scm/upgrade/package-info.java | 21 + .../ozone/container/common/TestEndPoint.java | 20 +- .../placement/TestContainerPlacement.java | 5 +- .../testutils/ReplicationNodeManagerMock.java | 23 +- .../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 63 +- .../main/java/org/apache/ozone/test/TestClock.java | 76 ++ .../hdds/scm/cli/ContainerOperationClient.java | 16 +- .../hdds/scm/cli/SafeModeWaitSubcommand.java | 3 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 22 + .../scm/cli/datanode/TestListInfoSubcommand.java | 13 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 16 + .../org/apache/hadoop/ozone/client/OzoneKey.java | 4 + .../ozone/client/io/BlockOutputStreamEntry.java | 3 +- .../ozone/client/protocol/ClientProtocol.java | 16 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 21 + hadoop-ozone/common/pom.xml | 4 + .../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 + .../apache/hadoop/ozone/om/OmUpgradeConfig.java | 82 ++ .../hadoop/ozone/om/exceptions/OMException.java | 7 + .../ozone/om/ha/OMFailoverProxyProvider.java | 67 +- .../hadoop/ozone/om/helpers}/OMNodeDetails.java | 86 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +- .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 11 + .../hadoop/ozone/om/helpers/WithObjectID.java | 6 +- .../ozone/om/protocol/OMInterServiceProtocol.java} | 28 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 105 ++ .../ozone/om/protocolPB/Hadoop3OmTransport.java | 2 +- .../OMInterServiceProtocolClientSideImpl.java | 121 +++ .../om/protocolPB/OMInterServiceProtocolPB.java | 38 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 111 +- .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 5 +- .../main/resources/ozone-version-info.properties | 3 +- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 31 + .../ozone/om/ha/TestOMFailoverProxyProvider.java | 7 +- .../om/helpers/TestOmKeyLocationInfoGroup.java | 3 +- hadoop-ozone/dev-support/checks/_lib.sh | 9 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 2 +- .../1.0.0/test.sh => ozone-mr/hadoop33/.env} | 13 +- .../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 ++ .../test.sh => ozone-mr/hadoop33/docker-config} | 20 +- .../compose/{upgrade => ozone-mr/hadoop33}/test.sh | 31 +- .../compose/{upgrade => ozone-om-prepare}/.env | 10 +- .../docker-compose.yaml | 75 +- .../{upgrade => ozone-om-prepare}/docker-config | 28 +- .../dist/src/main/compose/ozone-om-prepare/test.sh | 74 ++ .../src/main/compose/ozone/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozonesecure-mr/.env | 2 +- .../src/main/compose/ozonesecure/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +- .../dist/src/main/compose/upgrade/README.md | 95 +- .../src/main/compose/upgrade/{ => compose/ha}/.env | 7 +- .../upgrade/{ => compose/ha}/docker-compose.yaml | 120 ++- .../compose/upgrade/{ => compose/ha}/docker-config | 30 +- .../upgrade/{1.0.0/test.sh => compose/ha/load.sh} | 14 +- .../main/compose/upgrade/{ => compose/non-ha}/.env | 2 +- .../{ => compose/non-ha}/docker-compose.yaml | 99 +- .../upgrade/{ => compose/non-ha}/docker-config | 13 +- .../{0.5.0/test.sh => compose/non-ha/load.sh} | 13 +- hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 30 +- .../dist/src/main/compose/upgrade/testlib.sh | 135 ++- .../src/main/compose/upgrade/upgrade_to_current.sh | 51 - .../manual-upgrade/0.5.0-1.1.0/callback.sh} | 26 +- .../manual-upgrade/driver.sh} | 63 +- .../non-rolling-upgrade/1.0.0-1.1.0/callback.sh | 79 ++ .../upgrade/upgrades/non-rolling-upgrade/driver.sh | 101 ++ .../dist/src/main/compose/versions/0.5.0.sh | 4 - .../dist/src/main/compose/versions/1.0.0.sh | 4 - .../dist/src/main/compose/versions/1.1.0.sh | 4 - .../dist/src/main/compose/versions/README.md | 3 - hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 + .../license/bin/licenses/LICENSE-org.aspectj.html | 91 ++ .../LICENSE-org.reflections-reflections.txt | 14 + hadoop-ozone/dist/src/main/license/jar-report.txt | 69 +- .../src/main/smoketest/admincli/datanode.robot | 18 + .../main/smoketest/omha/om-cancel-prepare.robot | 43 + .../dist/src/main/smoketest/omha/om-prepare.robot | 49 + .../bucketcreate.robot => omha/om-prepared.robot} | 29 +- .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +- .../bucketcreate.robot => upgrade/check-mlv.robot} | 30 +- .../bucketcreate.robot => upgrade/finalize.robot} | 30 +- .../bucketcreate.robot => upgrade/generate.robot} | 26 +- .../bucketcreate.robot => upgrade/prepare.robot} | 25 +- .../bucketcreate.robot => upgrade/validate.robot} | 23 +- .../dist/src/shell/ozone/ozone-functions.sh | 2 +- .../dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} | 2 +- .../upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh | 0 .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 9 +- .../apache/hadoop/ozone/insight/LogSubcommand.java | 9 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 1067 ++++++++++++++++++++ .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 25 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 52 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 263 ++++- .../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 7 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 + .../hadoop/ozone/TestOzoneConfigurationFields.java | 2 + .../hadoop/ozone/TestStorageContainerManager.java | 29 +- .../ozone/client/rpc/TestBlockOutputStream.java | 13 +- .../rpc/TestBlockOutputStreamFlushDelay.java | 13 +- .../rpc/TestBlockOutputStreamWithFailures.java | 17 +- ...estBlockOutputStreamWithFailuresFlushDelay.java | 17 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 13 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 68 +- .../ozone/client/rpc/TestWatchForCommit.java | 3 +- .../ozone/container/TestContainerReplication.java | 81 +- .../apache/hadoop/ozone/container/TestHelper.java | 6 +- .../commandhandler/TestBlockDeletion.java | 266 +++-- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 254 +++++ .../org/apache/hadoop/ozone/om/TestOmAcls.java | 1 - .../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++ .../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 ++++ .../ozone/om/TestOzoneManagerConfiguration.java | 2 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 3 + .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 510 ++++++++++ .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 + .../ozone/recon/TestReconWithOzoneManagerFSO.java | 163 +++ .../TestSCMContainerPlacementPolicyMetrics.java | 2 +- .../dev-support/findbugsExcludeFile.xml | 6 + hadoop-ozone/interface-client/pom.xml | 62 +- .../src/main/proto/OmClientProtocol.proto | 95 +- .../src/main/proto/OmInterServiceProtocol.proto | 64 ++ hadoop-ozone/interface-storage/pom.xml | 2 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 13 + .../dev-support/findbugsExcludeFile.xml | 4 + hadoop-ozone/ozone-manager/pom.xml | 43 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 45 +- .../apache/hadoop/ozone/om/OMPolicyProvider.java | 4 + .../apache/hadoop/ozone/om/OMStarterInterface.java | 4 + .../java/org/apache/hadoop/ozone/om/OMStorage.java | 7 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 43 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 621 +++++++++--- .../hadoop/ozone/om/OzoneManagerPrepareState.java | 311 ++++++ .../hadoop/ozone/om/OzoneManagerStarter.java | 79 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 + .../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +- .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 11 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 15 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 405 +++++--- .../ozone/om/ratis/OzoneManagerStateMachine.java | 87 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 97 ++ .../hadoop/ozone/om/request/OMClientRequest.java | 7 +- .../om/request/bucket/OMBucketDeleteRequest.java | 1 + .../request/bucket/OMBucketSetPropertyRequest.java | 1 + .../om/request/bucket/acl/OMBucketAclRequest.java | 8 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 12 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 8 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 7 - .../om/request/file/OMDirectoryCreateRequest.java | 1 + .../ozone/om/request/file/OMFileCreateRequest.java | 7 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 1 + .../ozone/om/request/key/OMKeyDeleteRequest.java | 1 + .../ozone/om/request/key/OMKeyPurgeRequest.java | 1 + .../ozone/om/request/key/OMKeyRenameRequest.java | 1 + .../om/request/key/acl/OMKeyAddAclRequest.java | 1 + .../om/request/key/acl/OMKeyRemoveAclRequest.java | 1 + .../om/request/key/acl/OMKeySetAclRequest.java | 1 + .../key/acl/prefix/OMPrefixAddAclRequest.java | 1 + .../key/acl/prefix/OMPrefixRemoveAclRequest.java | 1 + .../key/acl/prefix/OMPrefixSetAclRequest.java | 1 + .../S3InitiateMultipartUploadRequest.java | 1 + .../S3MultipartUploadCommitPartRequest.java | 11 +- .../S3MultipartUploadCompleteRequest.java | 1 + .../om/request/s3/security/S3GetSecretRequest.java | 1 + .../security/OMCancelDelegationTokenRequest.java | 2 +- .../security/OMGetDelegationTokenRequest.java | 2 +- .../security/OMRenewDelegationTokenRequest.java | 1 + .../om/request/upgrade/OMCancelPrepareRequest.java | 97 ++ .../request/upgrade/OMFinalizeUpgradeRequest.java | 116 +++ .../ozone/om/request/upgrade/OMPrepareRequest.java | 257 +++++ .../security => request/upgrade}/package-info.java | 15 +- .../om/request/volume/OMVolumeDeleteRequest.java | 2 + .../om/request/volume/OMVolumeSetOwnerRequest.java | 1 + .../om/request/volume/OMVolumeSetQuotaRequest.java | 1 + .../hadoop/ozone/om/response/CleanupTableInfo.java | 10 +- .../om/response/key/OMOpenKeysDeleteRequest.java | 1 + .../om/response/s3/security/package-info.java | 2 +- .../response/upgrade/OMCancelPrepareResponse.java | 46 + .../upgrade/OMFinalizeUpgradeResponse.java | 58 ++ .../om/response/upgrade/OMPrepareResponse.java | 60 ++ .../{s3/security => upgrade}/package-info.java | 15 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 18 +- .../BelongsToLayoutVersion.java} | 23 +- .../DisallowedUntilLayoutVersion.java} | 31 +- .../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 80 ++ .../ozone/om/upgrade/OMLayoutFeatureAspect.java | 114 +++ .../ozone/om/upgrade/OMLayoutVersionManager.java | 210 ++++ .../ozone/om/upgrade/OMUpgradeFinalizer.java | 50 + .../hadoop/ozone/om/upgrade/OmUpgradeAction.java | 19 +- .../hadoop/ozone/om/upgrade/UpgradeActionOm.java | 37 + .../s3/security => upgrade}/package-info.java | 7 +- .../OMInterServiceProtocolServerSideImpl.java | 90 ++ ...OzoneManagerProtocolServerSideTranslatorPB.java | 97 +- .../protocolPB/OzoneManagerRequestHandler.java | 70 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 9 + .../hadoop/ozone/om/TestOzoneManagerStarter.java | 34 + .../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +- .../om/ratis/TestOzoneManagerRatisServer.java | 8 +- .../om/ratis/TestOzoneManagerStateMachine.java | 108 +- .../ozone/om/request/bucket/TestBucketRequest.java | 5 + .../ozone/om/request/key/TestOMKeyRequest.java | 17 + .../TestS3MultipartUploadCompleteRequest.java | 19 +- .../upgrade/TestOMCancelPrepareRequest.java | 110 ++ .../request/volume/TestOMVolumeDeleteRequest.java | 2 +- .../om/request/volume/TestOMVolumeRequest.java | 9 +- .../ozone/om/response/TestCleanupTableInfo.java | 26 +- .../hadoop/ozone/om/upgrade/MockOmRequest.java | 23 +- .../ozone/om/upgrade/OMLayoutFeatureUtil.java | 66 ++ .../om/upgrade/TestOMLayoutFeatureAspect.java | 99 ++ .../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 290 ++++++ .../ozone/om/upgrade/TestOMVersionManager.java | 195 ++++ .../TestOmVersionManagerRequestFactory.java | 111 ++ .../om/upgrade/TestOzoneManagerPrepareState.java | 283 ++++++ .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +- .../hadoop/fs/ozone/Hadoop27RpcTransport.java | 2 +- hadoop-ozone/recon/pom.xml | 5 + .../hadoop/ozone/recon/ReconControllerModule.java | 2 + .../org/apache/hadoop/ozone/recon/ReconServer.java | 11 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 31 + .../ozone/recon/api/ClusterStateEndpoint.java | 3 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 781 ++++++++++++++ .../hadoop/ozone/recon/api/NodeEndpoint.java | 15 +- .../hadoop/ozone/recon/api/types/DUResponse.java | 108 ++ .../ozone/recon/api/types/DatanodeMetadata.java | 13 + .../hadoop/ozone/recon/api/types/EntityType.java} | 15 +- .../api/types/FileSizeDistributionResponse.java | 61 ++ .../hadoop/ozone/recon/api/types/NSSummary.java | 57 +- .../recon/api/types/NamespaceSummaryResponse.java | 105 ++ .../ozone/recon/api/types/QuotaUsageResponse.java | 69 ++ .../ozone/recon/api/types/ResponseStatus.java | 13 +- .../hadoop/ozone/recon/codec/NSSummaryCodec.java | 56 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 139 ++- .../scm/ReconStorageContainerManagerFacade.java | 10 +- .../recon/spi/ReconNamespaceSummaryManager.java | 4 +- .../spi/impl/ReconNamespaceSummaryManagerImpl.java | 7 +- .../ozone/recon/tasks/FileSizeCountTask.java | 24 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 302 ++++++ .../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 1 + .../ozone/recon/tasks/OMDBUpdatesHandler.java | 2 +- .../src/views/datanodes/datanodes.tsx | 8 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 86 ++ .../hadoop/ozone/recon/api/TestEndpoints.java | 13 +- .../ozone/recon/api/TestNSSummaryEndpoint.java | 498 +++++++++ .../ozone/recon/api/TestOpenContainerCount.java | 7 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 3 +- .../scm/AbstractReconContainerManagerTest.java | 13 +- ...TestReconIncrementalContainerReportHandler.java | 14 +- .../ozone/recon/scm/TestReconNodeManager.java | 57 +- .../ozone/recon/scm/TestReconPipelineManager.java | 23 +- .../impl/TestReconNamespaceSummaryManagerImpl.java | 22 +- .../recon/tasks/TestContainerKeyMapperTask.java | 17 +- .../ozone/recon/tasks/TestNSSummaryTask.java | 519 ++++++++++ hadoop-ozone/s3gateway/pom.xml | 5 + .../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 + .../hadoop/ozone/s3/OzoneClientProducer.java | 4 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 7 +- .../apache/hadoop/ozone/s3/util/ContinueToken.java | 15 +- .../hadoop/ozone/client/OzoneBucketStub.java | 20 + .../hadoop/ozone/s3/util/TestContinueToken.java | 20 + .../ozone/admin/om/CancelPrepareSubCommand.java | 53 + .../admin/om/FinalizationStatusSubCommand.java | 64 ++ .../ozone/admin/om/FinalizeUpgradeSubCommand.java | 171 ++++ .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 44 +- .../hadoop/ozone/admin/om/PrepareSubCommand.java | 173 ++++ .../admin/scm/FinalizationScmStatusSubcommand.java | 51 + .../admin/scm/FinalizeScmUpgradeSubcommand.java | 156 +++ .../admin/scm/FinalizeUpgradeCommandUtil.java | 92 ++ .../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 4 +- .../ozone/audit/parser/common/DatabaseHelper.java | 4 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +- .../freon/FollowerAppendLogEntryGenerator.java | 4 +- .../hadoop/ozone/freon/RandomKeyGenerator.java | 19 +- .../hadoop/ozone/freon/StreamingGenerator.java | 70 +- .../containergenerator/GeneratorDatanode.java | 60 +- .../containergenerator/TestGeneratorDatanode.java | 71 ++ pom.xml | 15 +- 471 files changed, 22622 insertions(+), 2774 deletions(-) diff --cc hadoop-ozone/common/pom.xml index 873c6ef,67217c2..c994932 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@@ -73,13 -73,9 +73,17 @@@ https://maven.apache.org/xsd/maven-4.0. <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>hdds-server-framework</artifactId> + </dependency> ++ <dependency> + <groupId>commons-net</groupId> + <artifactId>commons-net</artifactId> + </dependency> + <dependency> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + </dependency> </dependencies> <build> diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 2efdb19,953aa23..149aaf0 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@@ -237,12 -237,11 +237,19 @@@ public class OMException extends IOExce DIRECTORY_NOT_EMPTY, + PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED, + REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED, + UPDATE_LAYOUT_VERSION_FAILED, + LAYOUT_FEATURE_FINALIZATION_FAILED, + PREPARE_FAILED, - NOT_SUPPORTED_OPERATION_WHEN_PREPARED ++ NOT_SUPPORTED_OPERATION_WHEN_PREPARED, ++ + TENANT_NOT_FOUND, + TENANT_ALREADY_EXISTS, + INVALID_TENANT_NAME, + + TENANT_USER_NOT_FOUND, + TENANT_USER_ALREADY_EXISTS, + INVALID_TENANT_USER_NAME } } diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 1c3f9be,41604a3..9d741a4 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -357,13 -348,12 +373,20 @@@ enum Status DIRECTORY_NOT_EMPTY = 68; - TENANT_NOT_FOUND = 69; - TENANT_ALREADY_EXISTS = 70; - INVALID_TENANT_NAME = 71; + PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED = 69; + REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED = 70; + UPDATE_LAYOUT_VERSION_FAILED = 71; + LAYOUT_FEATURE_FINALIZATION_FAILED = 72; + PREPARE_FAILED = 73; + NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74; + - TENANT_USER_NOT_FOUND = 72; - TENANT_USER_ALREADY_EXISTS = 73; - INVALID_TENANT_USER_NAME = 74; ++ TENANT_NOT_FOUND = 75; ++ TENANT_ALREADY_EXISTS = 76; ++ INVALID_TENANT_NAME = 77; ++ ++ TENANT_USER_NOT_FOUND = 78; ++ TENANT_USER_ALREADY_EXISTS = 79; ++ INVALID_TENANT_USER_NAME = 80; } /** diff --cc hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 519bb08,dba75b4..760cc00 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@@ -350,21 -354,13 +357,27 @@@ public interface OMMetadataManager exte Table<String, TransactionInfo> getTransactionInfoTable(); + Table<String, String> getTenantUserTable(); + + Table<String, OmDBAccessIdInfo> getTenantAccessIdTable(); + + Table<String, OmDBKerberosPrincipalInfo> getPrincipalToAccessIdsTable(); + + Table<String, OmDBTenantInfo> getTenantStateTable(); + + Table<String, String> getTenantGroupTable(); + + Table<String, String> getTenantRoleTable(); + + Table<String, String> getTenantPolicyTable(); + /** + * Gets the OM Meta table. + * @return meta table reference. + */ + Table<String, String> getMetaTable(); + + /** * Returns number of rows in a table. This should not be used for very * large tables. * @param table diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8d8257b,a854ca4..6c81c93 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@@ -189,17 -171,27 +189,44 @@@ public class OmMetadataManagerImpl impl public static final String DELETED_DIR_TABLE = "deletedDirectoryTable"; public static final String TRANSACTION_INFO_TABLE = "transactionInfoTable"; + public static final String META_TABLE = "metaTable"; + // Tables for S3 multi-tenancy + public static final String TENANT_USER_TABLE = "tenantUserTable"; + public static final String TENANT_ACCESS_ID_TABLE = "tenantAccessIdTable"; + public static final String PRINCIPAL_TO_ACCESS_IDS_TABLE = + "principalToAccessIdsTable"; + public static final String TENANT_STATE_TABLE = "tenantStateTable"; + public static final String TENANT_GROUP_TABLE = "tenantGroupTable"; + public static final String TENANT_ROLE_TABLE = "tenantRoleTable"; + public static final String TENANT_POLICY_TABLE = "tenantPolicyTable"; + + static final String[] ALL_TABLES = new String[] { + USER_TABLE, + VOLUME_TABLE, + BUCKET_TABLE, + KEY_TABLE, + DELETED_TABLE, + OPEN_KEY_TABLE, + MULTIPARTINFO_TABLE, + S3_SECRET_TABLE, + DELEGATION_TOKEN_TABLE, + PREFIX_TABLE, + TRANSACTION_INFO_TABLE, + DIRECTORY_TABLE, + FILE_TABLE, + DELETED_DIR_TABLE, + OPEN_FILE_TABLE, - META_TABLE ++ META_TABLE, ++ TENANT_USER_TABLE, ++ TENANT_ACCESS_ID_TABLE, ++ PRINCIPAL_TO_ACCESS_IDS_TABLE, ++ TENANT_STATE_TABLE, ++ TENANT_GROUP_TABLE, ++ TENANT_ROLE_TABLE, ++ TENANT_POLICY_TABLE + }; + private DBStore store; private final OzoneManagerLock lock; @@@ -219,15 -211,7 +246,17 @@@ private Table fileTable; private Table openFileTable; private Table transactionInfoTable; + private Table metaTable; ++ + // Tables for S3 multi-tenancy + private Table tenantUserTable; + private Table tenantAccessIdTable; + private Table principalToAccessIdsTable; + private Table tenantStateTable; + private Table tenantGroupTable; + private Table tenantRoleTable; + private Table tenantPolicyTable; + private boolean isRatisEnabled; private boolean ignorePipelineinKey; private Table deletedDirTable; @@@ -426,13 -415,7 +460,14 @@@ .addTable(OPEN_FILE_TABLE) .addTable(DELETED_DIR_TABLE) .addTable(TRANSACTION_INFO_TABLE) + .addTable(META_TABLE) + .addTable(TENANT_USER_TABLE) + .addTable(TENANT_ACCESS_ID_TABLE) + .addTable(PRINCIPAL_TO_ACCESS_IDS_TABLE) + .addTable(TENANT_STATE_TABLE) + .addTable(TENANT_GROUP_TABLE) + .addTable(TENANT_ROLE_TABLE) + .addTable(TENANT_POLICY_TABLE) .addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec()) .addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true)) .addCodec(RepeatedOmKeyInfo.class, @@@ -519,42 -501,8 +554,45 @@@ String.class, TransactionInfo.class); checkTableStatus(transactionInfoTable, TRANSACTION_INFO_TABLE); + metaTable = this.store.getTable(META_TABLE, String.class, String.class); + checkTableStatus(metaTable, META_TABLE); ++ + // tenant user name -> tenant name string + tenantUserTable = this.store.getTable(TENANT_USER_TABLE, + String.class, String.class); + checkTableStatus(tenantUserTable, TENANT_USER_TABLE); + + // tenantId -> OmDBAccessIdInfo (tenantId, secret, Kerberos principal) + tenantAccessIdTable = this.store.getTable(TENANT_ACCESS_ID_TABLE, + String.class, OmDBAccessIdInfo.class); + checkTableStatus(tenantAccessIdTable, TENANT_ACCESS_ID_TABLE); + + // Kerberos principal -> OmDBKerberosPrincipalInfo (A list of accessIds) + principalToAccessIdsTable = this.store.getTable( + PRINCIPAL_TO_ACCESS_IDS_TABLE, + String.class, OmDBKerberosPrincipalInfo.class); + checkTableStatus(principalToAccessIdsTable, PRINCIPAL_TO_ACCESS_IDS_TABLE); + + // tenant name -> tenant (tenant states) + tenantStateTable = this.store.getTable(TENANT_STATE_TABLE, + String.class, OmDBTenantInfo.class); + checkTableStatus(tenantStateTable, TENANT_STATE_TABLE); + + // tenant user name -> list of tenant groups the user belongs to + tenantGroupTable = this.store.getTable(TENANT_GROUP_TABLE, + String.class, String.class /* TODO: Use custom list */); + checkTableStatus(tenantGroupTable, TENANT_GROUP_TABLE); + + // tenant user name -> list of roles in a tenant. e.g. admin for "finance" + // TODO: Placeholder. Unused in the prototype. + tenantRoleTable = this.store.getTable(TENANT_ROLE_TABLE, + String.class, String.class /* TODO: Use custom list */); + checkTableStatus(tenantRoleTable, TENANT_ROLE_TABLE); + + // tenant policy name -> list of tenant policies + tenantPolicyTable = this.store.getTable(TENANT_POLICY_TABLE, + String.class, String.class /* TODO: Use custom list */); + checkTableStatus(tenantPolicyTable, TENANT_POLICY_TABLE); } /** @@@ -1290,41 -1238,10 +1328,46 @@@ } @Override + public Table<String, String> getMetaTable() { + return metaTable; + } + ++ @Override + public Table<String, String> getTenantUserTable() { + return tenantUserTable; + } + + @Override + public Table<String, OmDBAccessIdInfo> getTenantAccessIdTable() { + return tenantAccessIdTable; + } + + @Override + public Table<String, OmDBKerberosPrincipalInfo> + getPrincipalToAccessIdsTable() { + return principalToAccessIdsTable; + } + + @Override + public Table<String, OmDBTenantInfo> getTenantStateTable() { + return tenantStateTable; + } + + @Override + public Table<String, String> getTenantGroupTable() { + return tenantGroupTable; + } + + @Override + public Table<String, String> getTenantRoleTable() { + return tenantRoleTable; + } + + @Override + public Table<String, String> getTenantPolicyTable() { + return tenantPolicyTable; + } + /** * Update store used by subclass. * diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 36d605c,393632e..c5c6859 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -541,11 -574,9 +576,11 @@@ public final class OzoneManager extend * When OM state is reloaded, these services are re-initialized with the * new OM state. */ - private void instantiateServices() throws IOException { + private void instantiateServices(boolean withNewSnapshot) throws IOException { metadataManager = new OmMetadataManagerImpl(configuration); + multiTenantManagr = new OMMultiTenantManagerImpl(metadataManager, + configuration); volumeManager = new VolumeManagerImpl(metadataManager, configuration); bucketManager = new BucketManagerImpl(metadataManager, getKmsProvider(), isRatisEnabled); diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index c1d53e6,4b2ac49..11694c6 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@@ -180,75 -177,14 +180,83 @@@ public class OMDBDefinition implements String.class, new StringCodec(), OmKeyInfo.class, new OmKeyInfoCodec(true)); + public static final DBColumnFamilyDefinition<String, String> + META_TABLE = new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.META_TABLE, + String.class, + new StringCodec(), + String.class, + new StringCodec()); + + // Tables for S3 multi-tenancy + + // TODO: this table will be removed with the disappearance of CreateUser API. + public static final DBColumnFamilyDefinition<String, String> + TENANT_USER_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_USER_TABLE, + String.class, + new StringCodec(), + String.class, + new StringCodec()); + + public static final DBColumnFamilyDefinition<String, OmDBAccessIdInfo> + TENANT_ACCESS_ID_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_ACCESS_ID_TABLE, + String.class, // accessId + new StringCodec(), + OmDBAccessIdInfo.class, // tenantId, secret, principal + new OmDBAccessIdInfoCodec()); + + public static final DBColumnFamilyDefinition<String, + OmDBKerberosPrincipalInfo> + PRINCIPAL_TO_ACCESS_IDS_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.PRINCIPAL_TO_ACCESS_IDS_TABLE, + String.class, // Kerberos principal + new StringCodec(), + OmDBKerberosPrincipalInfo.class, // List of accessIds + new OmDBKerberosPrincipalInfoCodec()); + + public static final DBColumnFamilyDefinition<String, OmDBTenantInfo> + TENANT_STATE_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_STATE_TABLE, + String.class, + new StringCodec(), + OmDBTenantInfo.class, + new OmDBTenantInfoCodec()); + + public static final DBColumnFamilyDefinition<String, String> + TENANT_GROUP_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_GROUP_TABLE, + String.class, + new StringCodec(), + String.class, + new StringCodec()); + + public static final DBColumnFamilyDefinition<String, String> + TENANT_ROLE_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_ROLE_TABLE, + String.class, + new StringCodec(), + String.class, + new StringCodec()); + + public static final DBColumnFamilyDefinition<String, String> + TENANT_POLICY_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.TENANT_POLICY_TABLE, + String.class, + new StringCodec(), + String.class, + new StringCodec()); + + // End tables for S3 multi-tenancy + @Override public String getName() { return OzoneConsts.OM_DB_NAME; @@@ -265,10 -201,7 +273,10 @@@ VOLUME_TABLE, OPEN_KEY_TABLE, KEY_TABLE, BUCKET_TABLE, MULTIPART_INFO_TABLE, PREFIX_TABLE, DTOKEN_TABLE, S3_SECRET_TABLE, TRANSACTION_INFO_TABLE, DIRECTORY_TABLE, - FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, - FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE}; ++ FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE, + TENANT_USER_TABLE, TENANT_ACCESS_ID_TABLE, + PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE, TENANT_GROUP_TABLE, + TENANT_ROLE_TABLE, TENANT_POLICY_TABLE }; } } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 28ca13f,e655b28..9a874c9 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@@ -228,20 -244,14 +250,26 @@@ public final class OzoneManagerRatisUti return new S3GetSecretRequest(omRequest); case RecoverTrash: return new OMTrashRecoverRequest(omRequest); + case FinalizeUpgrade: + return new OMFinalizeUpgradeRequest(omRequest); + case Prepare: + return new OMPrepareRequest(omRequest); + case CancelPrepare: + return new OMCancelPrepareRequest(omRequest); case RevokeS3Secret: return new S3RevokeSecretRequest(omRequest); + case CreateTenant: + return new OMTenantCreateRequest(omRequest); + case ModifyTenant: + return new OMTenantModifyRequest(omRequest); + case DeleteTenant: + return new OMTenantDeleteRequest(omRequest); + case CreateTenantUser: + return new OMTenantUserCreateRequest(omRequest); + case ModifyTenantUser: + return new OMTenantUserModifyRequest(omRequest); + case DeleteTenantUser: + return new OMTenantUserDeleteRequest(omRequest); default: throw new IllegalStateException("Unrecognized write command " + "type request" + cmdType); diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 247a9ca,7a25634..7983ae8 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@@ -132,27 -132,15 +133,22 @@@ public class OzoneManagerProtocolServer if (OmUtils.isReadOnly(request)) { return submitReadRequestToOM(request); } else { - raftServerStatus = omRatisServer.checkLeaderStatus(); - if (raftServerStatus == LEADER_AND_READY) { - try { - omClientRequest = - OzoneManagerRatisUtils.createClientRequest(request); - request = omClientRequest.preExecute(ozoneManager); - } catch (IOException ex) { - // As some of the preExecute returns error. So handle here. - if (omClientRequest != null) { - omClientRequest.handleRequestFailure(ozoneManager); - } - return createErrorResponse(request, ex); - } - OMResponse response = submitRequestToRatis(request); - if (!response.getSuccess() && omClientRequest != null) { + checkLeaderStatus(); + try { - OMClientRequest omClientRequest = createClientRequest(request); ++ omClientRequest = createClientRequest(request); + request = omClientRequest.preExecute(ozoneManager); + } catch (IOException ex) { + // As some of the preExecute returns error. So handle here. ++ if (omClientRequest != null) { + omClientRequest.handleRequestFailure(ozoneManager); + } - return response; - } else { - throw createLeaderErrorException(raftServerStatus); + return createErrorResponse(request, ex); } - return submitRequestToRatis(request); ++ OMResponse response = submitRequestToRatis(request); ++ if (!response.getSuccess() && omClientRequest != null) { ++ omClientRequest.handleRequestFailure(ozoneManager); ++ } ++ return response; } } else { return submitRequestDirectlyToOM(request); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
