This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-4944 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7f818ab4460917e3c0d2661bbb6264b0465e6526 Merge: 8516765 87b4971 Author: Siyao Meng <[email protected]> AuthorDate: Wed Sep 22 14:51:07 2021 -0700 Merge remote-tracking branch 'asf/master' into HDDS-4944 Change-Id: Ic10eae75c7a09165f8bc3be65465f714d26f4a4e .github/workflows/post-commit.yml | 6 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 + .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 20 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 2 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 24 +- .../protocol/StorageContainerLocationProtocol.java | 4 +- .../apache/hadoop/hdds/tracing/TraceAllMethod.java | 23 +- .../apache/hadoop/hdds/tracing/TracingUtil.java | 6 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 - .../upgrade/AbstractLayoutVersionManager.java | 158 +++-- .../common/src/main/resources/ozone-default.xml | 45 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 33 +- .../hadoop/hdds/tracing/TestTraceAllMethod.java | 68 ++ .../hadoop/hdds/tracing/TestTracingUtil.java | 49 ++ .../apache/hadoop/ozone/HddsDatanodeService.java | 5 +- .../container/common/helpers/ContainerUtils.java | 34 +- .../ozone/container/common/impl/ContainerSet.java | 3 + .../container/common/impl/HddsDispatcher.java | 1 + .../common/interfaces/ContainerDispatcher.java | 7 +- .../common/statemachine/DatanodeConfiguration.java | 21 + .../common/statemachine/DatanodeStateMachine.java | 6 +- .../CloseContainerCommandHandler.java | 2 + .../ClosePipelineCommandHandler.java | 13 +- .../commandhandler/DeleteBlocksCommandHandler.java | 4 +- .../states/endpoint/VersionEndpointTask.java | 5 +- .../server/ratis/ContainerStateMachine.java | 11 +- .../transport/server/ratis/XceiverServerRatis.java | 3 +- .../container/common/utils/HddsVolumeUtil.java | 72 +- .../ozone/container/common/volume/HddsVolume.java | 2 +- .../container/common/volume/MutableVolumeSet.java | 13 - .../container/keyvalue/KeyValueContainer.java | 12 +- .../container/keyvalue/KeyValueContainerCheck.java | 4 +- .../ozone/container/keyvalue/KeyValueHandler.java | 52 +- .../container/keyvalue/helpers/ChunkUtils.java | 18 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 9 +- .../keyvalue/impl/FilePerBlockStrategy.java | 9 +- .../container/ozoneimpl/ContainerController.java | 24 +- .../ozone/container/ozoneimpl/ContainerReader.java | 161 ++--- .../ozone/container/ozoneimpl/OzoneContainer.java | 5 +- .../replication/GrpcReplicationClient.java | 4 +- .../container/replication/ReplicationServer.java | 7 +- .../replication/SimpleContainerDownloader.java | 20 +- .../upgrade/DataNodeUpgradeFinalizer.java | 5 +- .../upgrade/DatanodeMetadataFeatures.java | 53 -- .../ScmHAFinalizeUpgradeActionDatanode.java | 129 ++++ .../ScmHAFirstUpgradeLayoutChangeAction.java | 45 -- .../upgrade/VersionedDatanodeFeatures.java | 145 ++++ .../hadoop/ozone/container/common/ScmTestMock.java | 9 + .../ozone/container/common/TestContainerCache.java | 23 +- .../common/TestKeyValueContainerData.java | 8 +- .../common/impl/TestContainerDataYaml.java | 41 +- .../TestCloseContainerCommandHandler.java | 33 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 741 +++++++++++++++++++++ hadoop-hdds/docs/content/concept/OzoneManager.md | 2 +- .../docs/content/concept/OzoneManager.zh.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 4 +- hadoop-hdds/docs/content/interface/ReconApi.md | 238 ++++++- hadoop-hdds/docs/content/interface/ReconApi.zh.md | 2 +- .../docs/themes/ozonedoc/layouts/index.html | 2 + ...inerLocationProtocolClientSideTranslatorPB.java | 19 +- .../hadoop/hdds/server/http/HttpServer2.java | 17 +- .../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++ .../src/main/proto/ScmAdminProtocol.proto | 2 + .../src/main/proto/DatanodeClientProtocol.proto | 1 + .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 13 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 12 +- .../hdds/scm/block/SCMBlockDeletingService.java | 9 +- .../container/AbstractContainerReportHandler.java | 6 +- .../scm/container/CloseContainerEventHandler.java | 4 +- .../hdds/scm/container/ContainerManager.java | 144 ++-- .../hdds/scm/container/ContainerManagerImpl.java | 73 +- .../hdds/scm/container/ContainerManagerV2.java | 190 ------ .../hdds/scm/container/ContainerReportHandler.java | 6 +- .../IncrementalContainerReportHandler.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 31 +- .../hdds/scm/container/SCMContainerManager.java | 672 ------------------- .../scm/container/balancer/ContainerBalancer.java | 50 +- .../balancer/ContainerBalancerConfiguration.java | 14 +- .../ContainerBalancerSelectionCriteria.java | 14 +- .../scm/container/balancer/FindTargetGreedy.java | 6 +- .../container/placement/metrics/SCMNodeStat.java | 23 - .../replication/ReplicationManagerMetrics.java | 44 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 7 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 18 + .../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 49 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 6 +- .../hdds/scm/node/NodeDecommissionManager.java | 8 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 47 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 20 + .../hdds/scm/pipeline/PipelineManagerImpl.java | 51 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 2 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 18 - .../pipeline/WritableRatisContainerProvider.java | 105 ++- ...inerLocationProtocolServerSideTranslatorPB.java | 15 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 10 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 10 +- .../scm/server/OzoneStorageContainerManager.java | 4 +- .../hdds/scm/server/SCMClientProtocolServer.java | 23 +- .../hadoop/hdds/scm/server/SCMConfigurator.java | 8 +- .../hdds/scm/server/StorageContainerManager.java | 54 +- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 12 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 6 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 4 +- .../container/TestCloseContainerEventHandler.java | 2 +- .../scm/container/TestContainerManagerImpl.java | 2 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../TestIncrementalContainerReportHandler.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 33 +- .../scm/container/TestSCMContainerManager.java | 324 --------- .../scm/container/TestUnknownContainerReport.java | 4 +- .../container/balancer/TestContainerBalancer.java | 19 +- .../TestSCMContainerPlacementRackAware.java | 27 + .../hdds/scm/node/TestContainerPlacement.java | 95 +-- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 8 +- .../hdds/scm/pipeline/MockPipelineManager.java | 20 + .../ozone/container/common/TestEndPoint.java | 4 +- .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 7 + .../scm/cli/ContainerBalancerStartSubcommand.java | 13 +- .../hdds/scm/cli/ContainerOperationClient.java | 7 +- .../hadoop/hdds/scm/cli/TopologySubcommand.java | 46 +- .../datanode/TestContainerBalancerSubCommand.java | 6 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 8 +- .../main/java/org/apache/hadoop/ozone/OFSPath.java | 45 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 36 +- hadoop-ozone/dev-support/checks/_lib.sh | 8 +- .../dist/src/main/compose/ozone-ha/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop27/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop31/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop32/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop33/test.sh | 2 + .../dist/src/main/compose/ozone-mr/test.sh | 3 + .../dist/src/main/compose/ozone/docker-config | 2 + .../compose/ozonesecure-ha/docker-compose.yaml | 2 + .../dist/src/main/compose/ozonesecure-ha/test.sh | 2 + .../dist/src/main/compose/ozonesecure-mr/test.sh | 2 + .../src/main/compose/ozonesecure/docker-config | 3 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 3 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 23 +- .../dist/src/main/compose/upgrade/README.md | 35 +- .../compose/upgrade/delete-and-regenerate-data.sh | 53 -- .../dist/src/main/compose/upgrade/delete-data.sh | 36 - hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 9 +- .../manual-upgrade/0.5.0-1.1.0/callback.sh | 40 -- .../upgrades/manual-upgrade}/README.md | 7 +- .../{1.0.0-1.1.0 => 1.1.0-1.2.0}/callback.sh | 17 +- .../upgrade/upgrades/non-rolling-upgrade/driver.sh | 31 +- .../dist/src/main/compose/versions/0.5.0.sh | 26 - .../dist/src/main/compose/versions/1.0.0.sh | 26 - .../dist/src/main/compose/versions/1.1.0.sh | 26 - hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 4 - hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 6 +- .../dist/src/main/smoketest/ozonefs/setup.robot | 2 +- .../dist/src/main/smoketest/recon/recon-api.robot | 53 +- .../dist/src/main/smoketest/topology/cli.robot | 6 + hadoop-ozone/integration-test/pom.xml | 5 + .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 30 +- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 69 +- .../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 8 +- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 35 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 4 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 4 +- .../TestContainerStateManagerIntegration.java | 140 ++-- .../metrics/TestSCMContainerManagerMetrics.java | 4 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +- .../hdds/scm/pipeline/TestPipelineClose.java | 4 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 41 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +- .../hadoop/ozone/MiniOzoneClusterProvider.java | 286 ++++++++ .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 21 +- .../ozone/TestContainerBalancerOperations.java | 8 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +- .../client/rpc/read/TestChunkInputStream.java | 13 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 44 -- .../ozone/client/rpc/read/TestKeyInputStream.java | 58 +- .../ozone/container/TestContainerReplication.java | 47 -- .../apache/hadoop/ozone/container/TestHelper.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 8 +- .../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 ---- .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +- .../TestDatanodeHddsVolumeFailureToleration.java | 40 +- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 3 - .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 28 +- .../ozone/om/TestOzoneManagerConfiguration.java | 2 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 29 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 17 + .../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 24 +- .../ozone/om/TestOzoneManagerHAWithData.java | 32 +- .../ozone/om/TestOzoneManagerListVolumes.java | 117 ++-- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 7 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 10 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 4 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +- .../hadoop/ozone/scm/TestXceiverClientManager.java | 18 +- .../scm/node/TestDecommissionAndMaintenance.java | 128 ++-- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 72 ++ hadoop-ozone/ozone-manager/pom.xml | 10 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 26 +- .../request/bucket/OMBucketSetPropertyRequest.java | 9 +- .../ozone/om/request/file/OMFileCreateRequest.java | 3 +- .../request/file/OMFileCreateRequestWithFSO.java | 5 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 7 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 5 +- .../ozone/om/upgrade/OMLayoutVersionManager.java | 3 +- .../bucket/TestOMBucketSetPropertyRequest.java | 30 +- .../om/request/key/TestOMKeyCreateRequest.java | 70 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 15 +- .../ozone/om/request/key/TestOMKeyRequest.java | 7 +- .../volume/TestOMVolumeSetQuotaRequest.java | 28 +- .../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 4 +- .../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 + .../apache/hadoop/ozone/recon/ReconHttpServer.java | 5 +- .../hadoop/ozone/recon/ReconRestServletModule.java | 144 ++-- .../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 4 - .../apache/hadoop/ozone/recon/api/AdminOnly.java | 33 + .../hadoop/ozone/recon/api/ContainerEndpoint.java | 1 + .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 142 ++-- .../ozone/recon/api/filters/ReconAdminFilter.java | 113 ++++ .../ozone/recon/api/filters/ReconAuthFilter.java | 117 ++++ .../package-info.java} | 10 +- .../hadoop/ozone/recon/api/types/DUResponse.java | 14 + .../ozone/recon/api/types/ResponseStatus.java | 1 + .../ozone/recon/fsck/ContainerHealthTask.java | 6 +- .../recon/scm/ReconContainerReportHandler.java | 4 +- .../ozone/recon/scm/ReconDeadNodeHandler.java | 4 +- .../ReconIncrementalContainerReportHandler.java | 4 +- .../ozone/recon/scm/ReconPipelineManager.java | 12 +- .../scm/ReconStorageContainerManagerFacade.java | 4 +- .../webapps/recon/ozone-recon-web/api/db.json | 324 ++++++++- .../webapps/recon/ozone-recon-web/api/routes.json | 22 +- .../src/components/navBar/navBar.tsx | 5 + .../src/components/rightDrawer/rightDrawer.tsx | 72 ++ .../src/constants/breadcrumbs.constants.tsx | 3 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 5 + .../diskUsage/diskUsage.less} | 41 +- .../src/views/diskUsage/diskUsage.tsx | 444 ++++++++++++ .../src/views/overview/overview.tsx | 3 +- .../ozone/recon/api/TestNSSummaryEndpoint.java | 12 +- .../ozone/recon/api/filters/TestAdminFilter.java | 161 +++++ .../ozone/recon/fsck/TestContainerHealthTask.java | 6 +- .../ozone/admin/nssummary/DiskUsageSubCommand.java | 214 ++++++ .../admin/nssummary/FileSizeDistSubCommand.java | 129 ++++ .../ozone/admin/nssummary/NSSummaryAdmin.java | 137 ++++ .../ozone/admin/nssummary/NSSummaryCLIUtils.java | 174 +++++ .../admin/nssummary/QuotaUsageSubCommand.java | 117 ++++ .../ozone/admin/nssummary/SummarySubCommand.java | 115 ++++ .../ozone/admin/nssummary/package-info.java} | 14 +- .../admin/scm/GetScmRatisRolesSubcommand.java | 7 +- .../apache/hadoop/ozone/debug/DatanodeLayout.java | 110 --- .../ozone/shell/volume/CreateVolumeHandler.java | 4 +- pom.xml | 5 +- 260 files changed, 6683 insertions(+), 3399 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
