This is an automated email from the ASF dual-hosted git repository. pifta pushed a commit to branch HDDS-5447-httpfs in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 56f104c71eb0cef202267f1adefd4e7b7f5a5886 Merge: 0f3e560 b4a785c Author: Istvan Fajth <[email protected]> AuthorDate: Tue Nov 16 12:12:56 2021 +0100 Merge branch 'master' into HDDS-5447-httpfs .github/ci.md | 102 +++ .github/workflows/post-commit.yml | 6 +- CONTRIBUTING.md | 4 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 52 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 + .../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +- .../storage/DummyBlockInputStreamWithRetry.java | 14 +- .../hdds/scm/storage/TestBlockInputStream.java | 32 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 51 ++ .../java/org/apache/hadoop/hdds/NodeDetails.java | 0 .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 23 +- .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 10 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +- .../hadoop/hdds/scm/container/ContainerInfo.java | 4 + .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 31 +- .../hadoop/hdds/scm/net/NetworkTopology.java | 3 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 5 +- .../protocol/StorageContainerLocationProtocol.java | 24 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 5 - .../apache/hadoop/hdds/tracing/TraceAllMethod.java | 23 +- .../apache/hadoop/hdds/tracing/TracingUtil.java | 6 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 - .../upgrade/AbstractLayoutVersionManager.java | 158 ++-- .../common/src/main/resources/ozone-default.xml | 84 +- .../hadoop/hdds/tracing/TestTraceAllMethod.java | 68 ++ .../hadoop/hdds/tracing/TestTracingUtil.java | 49 ++ .../apache/hadoop/ozone/HddsDatanodeService.java | 5 +- .../ozone/container/common/impl/ContainerSet.java | 3 + .../container/common/impl/HddsDispatcher.java | 1 + .../common/interfaces/ContainerDispatcher.java | 7 +- .../common/statemachine/DatanodeStateMachine.java | 19 +- .../common/statemachine/StateContext.java | 14 +- .../CloseContainerCommandHandler.java | 2 + .../ClosePipelineCommandHandler.java | 13 +- .../CreatePipelineCommandHandler.java | 10 +- .../RefreshVolumeUsageCommandHandler.java | 73 ++ .../states/endpoint/VersionEndpointTask.java | 5 +- .../common/transport/server/ratis/CSMMetrics.java | 10 + .../server/ratis/ContainerStateMachine.java | 9 +- .../transport/server/ratis/XceiverServerRatis.java | 15 +- .../container/common/utils/HddsVolumeUtil.java | 72 +- .../ozone/container/common/volume/HddsVolume.java | 2 +- .../container/common/volume/MutableVolumeSet.java | 17 +- .../container/common/volume/StorageVolume.java | 4 + .../common/volume/StorageVolumeChecker.java | 1 + .../ozone/container/common/volume/VolumeInfo.java | 13 +- .../ozone/container/common/volume/VolumeUsage.java | 3 + .../container/keyvalue/KeyValueContainer.java | 12 +- .../container/keyvalue/KeyValueContainerCheck.java | 4 +- .../ozone/container/keyvalue/KeyValueHandler.java | 34 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 7 +- .../metadata/DatanodeSchemaOneDBDefinition.java | 6 +- .../ozone/container/ozoneimpl/ContainerReader.java | 161 +--- .../ozone/container/ozoneimpl/OzoneContainer.java | 5 +- .../replication/DownloadAndImportReplicator.java | 5 +- .../replication/ReplicationSupervisor.java | 8 + .../replication/ReplicationSupervisorMetrics.java | 72 ++ .../replication/SimpleContainerDownloader.java | 27 +- .../upgrade/DataNodeUpgradeFinalizer.java | 5 +- .../upgrade/DatanodeMetadataFeatures.java | 53 -- .../ScmHAFinalizeUpgradeActionDatanode.java | 129 +++ .../ScmHAFirstUpgradeLayoutChangeAction.java | 45 -- .../upgrade/VersionedDatanodeFeatures.java | 145 ++++ .../hadoop/ozone/container/common/ScmTestMock.java | 9 + .../ozone/container/common/TestContainerCache.java | 23 +- .../common/TestKeyValueContainerData.java | 8 +- .../common/impl/TestContainerDataYaml.java | 6 +- .../common/volume/TestStorageVolumeChecker.java | 4 + .../replication/TestReplicationSupervisor.java | 59 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 741 +++++++++++++++++ hadoop-hdds/docs/content/feature/PrefixFSO.md | 16 + hadoop-hdds/docs/content/feature/Topology.md | 2 +- hadoop-hdds/docs/content/interface/CSI.md | 4 +- hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +- hadoop-hdds/docs/content/interface/Cli.md | 7 +- hadoop-hdds/docs/content/interface/ReconApi.md | 23 +- hadoop-hdds/docs/content/interface/ReconApi.zh.md | 2 +- .../docs/content/security/SecuringOzoneHTTP.md | 20 +- hadoop-hdds/docs/content/start/FromSource.md | 2 +- .../docs/static/design/s3-performance-new.png | Bin 33688 -> 34927 bytes .../docs/static/design/s3-performance-old.png | Bin 34927 -> 33688 bytes .../themes/ozonedoc/layouts/partials/navbar.html | 6 +- .../themes/ozonedoc/layouts/partials/sidebar.html | 4 +- .../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 4 +- .../hdds/conf/DatanodeRatisServerConfig.java | 16 + ...inerLocationProtocolClientSideTranslatorPB.java | 31 +- .../certificate/client/DNCertificateClient.java | 5 - .../client/DefaultCertificateClient.java | 2 +- .../certificate/client/OMCertificateClient.java | 5 - .../certificate/client/SCMCertificateClient.java | 10 +- .../hadoop/hdds/security/x509/keys/KeyCodec.java | 31 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 16 +- .../hadoop/hdds/utils/db/RDBStoreIterator.java | 19 +- .../apache/hadoop/hdds/utils/db/TableIterator.java | 12 - .../apache/hadoop/hdds/utils/db/TypedTable.java | 18 - .../hdds/security/x509/keys/TestKeyCodec.java | 5 +- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 19 +- .../src/main/proto/ScmAdminProtocol.proto | 3 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 + .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 6 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 22 +- .../hdds/scm/block/SCMBlockDeletingService.java | 9 +- .../container/AbstractContainerReportHandler.java | 6 +- .../scm/container/CloseContainerEventHandler.java | 4 +- .../hdds/scm/container/ContainerManager.java | 144 ++-- .../hdds/scm/container/ContainerManagerImpl.java | 8 +- .../hdds/scm/container/ContainerManagerV2.java | 190 ----- .../hdds/scm/container/ContainerReportHandler.java | 6 +- .../hdds/scm/container/ContainerStateManager.java | 570 +++---------- .../scm/container/ContainerStateManagerImpl.java | 19 +- .../scm/container/ContainerStateManagerV2.java | 189 ----- .../IncrementalContainerReportHandler.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 34 +- .../hdds/scm/container/SCMContainerManager.java | 672 ---------------- .../scm/container/balancer/ContainerBalancer.java | 287 ++++--- .../balancer/ContainerBalancerConfiguration.java | 58 +- .../balancer/ContainerBalancerMetrics.java | 115 +-- .../ContainerBalancerSelectionCriteria.java | 14 +- .../scm/container/balancer/FindTargetGreedy.java | 6 +- .../algorithms/SCMContainerPlacementRackAware.java | 72 +- .../container/placement/metrics/SCMNodeStat.java | 23 - .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 11 +- .../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 24 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 26 +- .../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 65 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 6 +- .../hdds/scm/node/NodeDecommissionManager.java | 8 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 8 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 60 +- .../scm/pipeline/BackgroundPipelineCreator.java | 264 ++++-- .../scm/pipeline/BackgroundPipelineCreatorV2.java | 332 -------- .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 2 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 10 + .../hdds/scm/pipeline/PipelineManagerImpl.java | 29 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 30 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 6 +- .../hdds/scm/pipeline/PipelineStateManager.java | 271 ++----- ...erV2Impl.java => PipelineStateManagerImpl.java} | 70 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 29 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 20 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +- .../hadoop/hdds/scm/pipeline/StateManager.java | 127 --- .../pipeline/WritableRatisContainerProvider.java | 6 +- .../algorithms/DefaultLeaderChoosePolicy.java | 4 +- .../choose/algorithms/LeaderChoosePolicy.java | 8 +- .../algorithms/LeaderChoosePolicyFactory.java | 8 +- .../algorithms/MinLeaderCountChoosePolicy.java | 6 +- ...inerLocationProtocolServerSideTranslatorPB.java | 21 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 10 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 10 +- .../scm/server/OzoneStorageContainerManager.java | 4 +- .../hdds/scm/server/SCMClientProtocolServer.java | 70 +- .../hadoop/hdds/scm/server/SCMConfigurator.java | 8 +- .../hdds/scm/server/StorageContainerManager.java | 73 +- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 30 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 6 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 16 +- .../hdds/scm/container/SimpleMockNodeManager.java | 5 + .../container/TestCloseContainerEventHandler.java | 2 +- .../scm/container/TestContainerManagerImpl.java | 8 +- .../scm/container/TestContainerReportHandler.java | 269 ++++--- .../scm/container/TestContainerStateManager.java | 91 ++- .../TestIncrementalContainerReportHandler.java | 130 +-- .../hdds/scm/container/TestReplicationManager.java | 395 +++++---- .../scm/container/TestSCMContainerManager.java | 324 -------- .../scm/container/TestUnknownContainerReport.java | 51 +- .../container/balancer/TestContainerBalancer.java | 91 ++- .../hdds/scm/container/balancer/package-info.java | 24 + .../TestSCMContainerPlacementRackAware.java | 15 + .../hdds/scm/ha/TestReplicationAnnotation.java | 8 +- .../hdds/scm/node/TestContainerPlacement.java | 122 +-- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 8 +- .../hdds/scm/pipeline/MockPipelineManager.java | 33 +- .../scm/pipeline/MockRatisPipelineProvider.java | 6 +- .../TestPipelineDatanodesIntersection.java | 52 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 26 + .../scm/pipeline/TestPipelinePlacementPolicy.java | 73 +- ...ager.java => TestPipelineStateManagerImpl.java} | 237 ++++-- .../scm/pipeline/TestRatisPipelineProvider.java | 66 +- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 209 ----- .../scm/pipeline/TestSimplePipelineProvider.java | 48 +- .../choose/algorithms/TestLeaderChoosePolicy.java | 6 +- .../ozone/container/common/TestEndPoint.java | 4 +- .../testutils/ReplicationNodeManagerMock.java | 11 + .../scm/cli/ContainerBalancerStartSubcommand.java | 13 +- .../hdds/scm/cli/ContainerOperationClient.java | 12 +- .../hadoop/hdds/scm/cli/TopologySubcommand.java | 46 +- .../hdds/scm/cli/container/ListSubcommand.java | 6 +- .../datanode/TestContainerBalancerSubCommand.java | 6 +- .../org/apache/hadoop/ozone/client/BucketArgs.java | 8 +- .../ozone/client/io/BlockOutputStreamEntry.java | 268 +++++-- .../client/io/BlockOutputStreamEntryPool.java | 109 ++- .../hadoop/ozone/client/io/KeyOutputStream.java | 16 +- .../ozone/client/protocol/ClientProtocol.java | 19 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 55 +- hadoop-ozone/common/pom.xml | 4 - .../main/java/org/apache/hadoop/ozone/OFSPath.java | 48 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 27 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 11 +- .../hadoop/ozone/om/helpers/BucketLayout.java | 25 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 7 - .../ozone/om/protocol/OzoneManagerProtocol.java | 168 +++- .../apache/hadoop/ozone/om/protocol/S3Auth.java | 46 ++ .../om/protocolPB/OzoneManagerClientProtocol.java | 38 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 59 +- .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 41 +- .../org/apache/hadoop/ozone/csi/NodeService.java | 10 +- hadoop-ozone/dev-support/checks/_lib.sh | 8 +- hadoop-ozone/dist/README.md | 6 +- .../dist/src/main/compose/ozone-ha/test.sh | 2 + .../dist/src/main/compose/ozone-mr/common-config | 1 + .../src/main/compose/ozone-mr/hadoop27/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop31/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop32/test.sh | 2 + .../src/main/compose/ozone-mr/hadoop33/test.sh | 2 + .../dist/src/main/compose/ozone-mr/test.sh | 3 + .../compose/ozonesecure-ha/docker-compose.yaml | 2 + .../dist/src/main/compose/ozonesecure-ha/test.sh | 2 + .../dist/src/main/compose/ozonesecure-mr/test.sh | 2 + .../src/main/compose/ozonesecure/docker-config | 2 + hadoop-ozone/dist/src/main/compose/testlib.sh | 16 - .../dist/src/main/compose/upgrade/README.md | 35 +- .../compose/upgrade/delete-and-regenerate-data.sh | 53 -- .../dist/src/main/compose/upgrade/delete-data.sh | 36 - hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 9 +- .../manual-upgrade/0.5.0-1.1.0/callback.sh | 40 - .../upgrades/manual-upgrade}/README.md | 7 +- .../{1.0.0-1.1.0 => 1.1.0-1.2.0}/callback.sh | 17 +- .../upgrade/upgrades/non-rolling-upgrade/driver.sh | 31 +- .../dist/src/main/compose/versions/0.5.0.sh | 26 - .../dist/src/main/compose/versions/1.0.0.sh | 26 - .../dist/src/main/compose/versions/1.1.0.sh | 26 - hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 4 - hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +- .../src/main/smoketest/admincli/container.robot | 8 + .../dist/src/main/smoketest/basic/links.robot | 8 +- .../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 2 +- .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +- .../dist/src/main/smoketest/recon/recon-api.robot | 53 +- .../src/main/smoketest/s3/MultipartUpload.robot | 23 +- .../main/smoketest/security/ozone-secure-fs.robot | 2 +- .../dist/src/main/smoketest/topology/cli.robot | 6 + .../dev-support/findbugsExcludeFile.xml | 4 + .../ozone/TestDirectoryDeletingServiceWithFSO.java | 19 +- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 33 +- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 107 ++- .../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 29 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 44 +- .../fs/ozone/TestOzoneFileSystemMetrics.java | 3 + .../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +- .../fs/ozone/TestOzoneFileSystemWithFSO.java | 9 +- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 35 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 102 ++- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 7 - .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 4 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 4 +- .../TestContainerStateManagerIntegration.java | 138 ++-- .../metrics/TestSCMContainerManagerMetrics.java | 4 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +- .../hdds/scm/pipeline/TestPipelineClose.java | 4 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 41 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 31 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 + .../hadoop/ozone/MiniOzoneClusterProvider.java | 3 + .../ozone/TestContainerBalancerOperations.java | 8 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +- .../rpc/TestBlockOutputStreamWithFailures.java | 20 +- ...estBlockOutputStreamWithFailuresFlushDelay.java | 20 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 27 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 135 +++- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 100 ++- .../client/rpc/read/TestChunkInputStream.java | 13 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 44 - .../ozone/client/rpc/read/TestKeyInputStream.java | 58 +- .../apache/hadoop/ozone/container/TestHelper.java | 4 +- .../commandhandler/TestBlockDeletion.java | 10 +- .../container/ozoneimpl/TestOzoneContainer.java | 8 +- .../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 --- .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +- .../TestDatanodeHddsVolumeFailureToleration.java | 40 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 44 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +- .../apache/hadoop/ozone/om/TestObjectStore.java | 103 +++ .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 26 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 27 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 64 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 301 ++++--- .../ozone/om/TestOzoneManagerConfiguration.java | 2 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 114 ++- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +- .../ozone/om/TestOzoneManagerHAWithData.java | 98 +-- .../ozone/om/TestOzoneManagerHAWithFailover.java | 65 ++ .../ozone/om/TestOzoneManagerListVolumes.java | 117 +-- .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 28 +- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 7 +- .../om/ratis/TestOzoneManagerRatisRequest.java | 71 ++ .../TestOzoneHARatisLogParser.java} | 52 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 10 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 4 +- .../ozone/recon/TestReconWithOzoneManager.java | 19 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +- .../hadoop/ozone/scm/TestXceiverClientManager.java | 18 +- .../scm/node/TestDecommissionAndMaintenance.java | 94 +-- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 9 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 142 +++- .../src/main/proto/OmClientProtocol.proto | 11 + .../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +- .../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +- hadoop-ozone/ozone-manager/pom.xml | 10 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 21 +- .../hadoop/ozone/om/DirectoryDeletingService.java | 21 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 777 +++++++++++------- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 14 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 29 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 893 +++------------------ .../hadoop/ozone/om/TrashOzoneFileSystem.java | 7 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 + .../org/apache/hadoop/ozone/om/VolumeManager.java | 40 +- .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 300 ------- .../ozone/om/ratis/OzoneManagerRatisServer.java | 18 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 206 +++-- .../hadoop/ozone/om/request/OMClientRequest.java | 22 +- .../om/request/bucket/OMBucketCreateRequest.java | 11 +- .../request/bucket/OMBucketSetPropertyRequest.java | 9 +- .../om/request/file/OMDirectoryCreateRequest.java | 10 +- .../file/OMDirectoryCreateRequestWithFSO.java | 6 +- .../ozone/om/request/file/OMFileCreateRequest.java | 9 +- .../request/file/OMFileCreateRequestWithFSO.java | 9 +- .../ozone/om/request/file/OMFileRequest.java | 140 +++- .../om/request/key/OMAllocateBlockRequest.java | 24 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 95 ++- .../om/request/key/OMKeyCommitRequestWithFSO.java | 35 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 29 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 74 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 8 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 14 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 9 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 71 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 5 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 9 +- .../om/request/key/OMPathsPurgeRequestWithFSO.java | 6 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 47 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 5 +- .../S3InitiateMultipartUploadRequest.java | 17 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 6 +- .../multipart/S3MultipartUploadAbortRequest.java | 34 +- .../S3MultipartUploadAbortRequestWithFSO.java | 6 +- .../S3MultipartUploadCommitPartRequest.java | 23 +- .../S3MultipartUploadCommitPartRequestWithFSO.java | 7 +- .../S3MultipartUploadCompleteRequest.java | 35 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 6 +- .../ozone/om/request/upgrade/OMPrepareRequest.java | 9 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 5 +- .../hadoop/ozone/om/response/OMClientResponse.java | 26 + .../response/file/OMDirectoryCreateResponse.java | 6 +- .../response/file/OMFileCreateResponseWithFSO.java | 5 + .../om/response/key/OMAllocateBlockResponse.java | 4 +- .../key/OMAllocateBlockResponseWithFSO.java | 6 + .../ozone/om/response/key/OMKeyCommitResponse.java | 26 +- .../response/key/OMKeyCommitResponseWithFSO.java | 21 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 6 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +- .../response/key/OMKeyDeleteResponseWithFSO.java | 9 +- .../ozone/om/response/key/OMKeyRenameResponse.java | 12 +- .../response/key/OMKeyRenameResponseWithFSO.java | 14 +- .../om/response/key/OMKeysDeleteResponse.java | 3 +- .../om/response/key/OMKeysRenameResponse.java | 11 +- .../om/response/key/OMOpenKeysDeleteRequest.java | 5 +- .../om/response/key/OMOpenKeysDeleteResponse.java | 3 +- .../response/key/OMPathsPurgeResponseWithFSO.java | 9 +- .../om/response/key/OMTrashRecoverResponse.java | 2 +- .../om/response/key/acl/OMKeyAclResponse.java | 4 +- .../response/key/acl/OMKeyAclResponseWithFSO.java | 4 +- .../S3InitiateMultipartUploadResponse.java | 4 +- .../multipart/S3MultipartUploadAbortResponse.java | 4 +- .../S3MultipartUploadAbortResponseWithFSO.java | 6 + .../S3MultipartUploadCommitPartResponse.java | 4 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 6 + .../S3MultipartUploadCompleteResponse.java | 38 +- .../ozone/om/upgrade/OMLayoutVersionManager.java | 3 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 57 +- .../protocolPB/OzoneManagerRequestHandler.java | 37 +- .../security/OzoneBlockTokenSecretManager.java | 5 - .../hadoop/ozone/security/S3SecurityUtil.java | 82 ++ .../hadoop/ozone/om/TestBucketManagerImpl.java | 63 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 7 +- .../ozone/om/request/TestOMRequestUtils.java | 125 ++- .../bucket/TestOMBucketCreateRequestWithFSO.java | 5 + .../bucket/TestOMBucketSetPropertyRequest.java | 30 +- .../request/file/TestOMDirectoryCreateRequest.java | 53 +- .../file/TestOMDirectoryCreateRequestWithFSO.java | 91 ++- .../om/request/file/TestOMFileCreateRequest.java | 8 +- .../file/TestOMFileCreateRequestWithFSO.java | 26 +- .../om/request/key/TestOMAllocateBlockRequest.java | 7 +- .../key/TestOMAllocateBlockRequestWithFSO.java | 25 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 30 +- .../om/request/key/TestOMKeyAclRequestWithFSO.java | 15 +- .../om/request/key/TestOMKeyCommitRequest.java | 131 ++- .../request/key/TestOMKeyCommitRequestWithFSO.java | 19 +- .../om/request/key/TestOMKeyCreateRequest.java | 70 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 20 +- .../om/request/key/TestOMKeyDeleteRequest.java | 26 +- .../request/key/TestOMKeyDeleteRequestWithFSO.java | 26 +- .../om/request/key/TestOMKeyRenameRequest.java | 8 +- .../ozone/om/request/key/TestOMKeyRequest.java | 17 +- .../om/request/key/TestOMKeysDeleteRequest.java | 4 +- .../om/request/key/TestOMKeysRenameRequest.java | 20 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 10 +- .../TestS3InitiateMultipartUploadRequest.java | 30 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 14 +- .../s3/multipart/TestS3MultipartRequest.java | 21 +- .../TestS3MultipartUploadAbortRequest.java | 5 +- .../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +- .../TestS3MultipartUploadCommitPartRequest.java | 17 +- ...tS3MultipartUploadCommitPartRequestWithFSO.java | 19 +- .../TestS3MultipartUploadCompleteRequest.java | 19 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 28 +- .../volume/TestOMVolumeSetQuotaRequest.java | 28 +- .../file/TestOMDirectoryCreateResponse.java | 7 +- .../file/TestOMFileCreateResponseWithFSO.java | 13 +- .../response/key/TestOMAllocateBlockResponse.java | 12 +- .../key/TestOMAllocateBlockResponseWithFSO.java | 14 +- .../om/response/key/TestOMKeyCommitResponse.java | 46 +- .../key/TestOMKeyCommitResponseWithFSO.java | 18 +- .../om/response/key/TestOMKeyCreateResponse.java | 12 +- .../key/TestOMKeyCreateResponseWithFSO.java | 19 +- .../om/response/key/TestOMKeyDeleteResponse.java | 18 +- .../key/TestOMKeyDeleteResponseWithFSO.java | 14 +- .../om/response/key/TestOMKeyRenameResponse.java | 30 +- .../ozone/om/response/key/TestOMKeyResponse.java | 8 + .../om/response/key/TestOMKeysDeleteResponse.java | 9 +- .../om/response/key/TestOMKeysRenameResponse.java | 16 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 18 +- .../TestS3InitiateMultipartUploadResponse.java | 3 +- ...stS3InitiateMultipartUploadResponseWithFSO.java | 9 +- .../s3/multipart/TestS3MultipartResponse.java | 15 +- .../TestS3MultipartUploadAbortResponse.java | 11 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 15 +- ...stS3MultipartUploadCompleteResponseWithFSO.java | 100 ++- .../hadoop/ozone/security/acl/TestParentAcl.java | 21 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 25 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +- .../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 + hadoop-ozone/ozonefs-shaded/pom.xml | 2 + hadoop-ozone/pom.xml | 2 +- .../apache/hadoop/ozone/recon/ReconHttpServer.java | 1 + .../hadoop/ozone/recon/ReconRestServletModule.java | 144 ++-- .../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +- .../apache/hadoop/ozone/recon/api/AdminOnly.java | 37 +- .../hadoop/ozone/recon/api/ContainerEndpoint.java | 10 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 5 +- .../ozone/recon/api/filters/ReconAdminFilter.java | 113 +++ .../ozone/recon/api/filters/ReconAuthFilter.java | 117 +++ .../ozone/recon/api/filters/package-info.java | 22 + .../ozone/recon/fsck/ContainerHealthTask.java | 9 +- .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 3 + .../recon/scm/ReconContainerReportHandler.java | 4 +- .../ozone/recon/scm/ReconDeadNodeHandler.java | 4 +- .../ReconIncrementalContainerReportHandler.java | 4 +- .../ozone/recon/scm/ReconPipelineManager.java | 10 +- .../scm/ReconStorageContainerManagerFacade.java | 4 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +- .../ozone/recon/tasks/FileSizeCountTask.java | 8 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 25 +- .../ozone/recon/api/TestContainerEndpoint.java | 8 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 7 +- .../ozone/recon/api/TestNSSummaryEndpoint.java | 15 +- .../ozone/recon/api/filters/TestAdminFilter.java | 161 ++++ .../ozone/recon/fsck/TestContainerHealthTask.java | 6 +- .../recovery/TestReconOmMetadataManagerImpl.java | 25 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 17 +- .../recon/tasks/TestContainerKeyMapperTask.java | 9 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 9 +- .../ozone/recon/tasks/TestNSSummaryTask.java | 16 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 30 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 77 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 + .../hadoop/ozone/s3/endpoint/EndpointBase.java | 29 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 40 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 + .../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +- .../s3/endpoint/TestMultipartUploadWithCopy.java | 219 +++-- .../ozone/admin/om/FinalizeUpgradeSubCommand.java | 5 +- .../admin/scm/FinalizeScmUpgradeSubcommand.java | 5 +- .../admin/scm/GetScmRatisRolesSubcommand.java | 7 +- .../hadoop/ozone/debug/DBDefinitionFactory.java | 17 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 80 +- .../apache/hadoop/ozone/debug/DatanodeLayout.java | 110 --- .../apache/hadoop/ozone/debug/PrefixParser.java | 16 +- .../ozone/debug/container/ContainerCommands.java | 16 +- .../ozone/freon/ClosedContainerReplicator.java | 10 +- .../containergenerator/GeneratorDatanode.java | 23 +- .../apache/hadoop/ozone/fsck/ContainerMapper.java | 7 +- .../ozone/genesis/BenchMarkOMKeyAllocation.java | 137 ---- .../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +- .../hadoop/ozone/segmentparser/RatisLogParser.java | 4 +- .../ozone/segmentparser/SCMRatisLogParser.java | 44 + .../org/apache/hadoop/ozone/shell/Handler.java | 17 + .../ozone/shell/bucket/CreateBucketHandler.java | 10 +- .../ozone/shell/bucket/ListBucketHandler.java | 10 +- .../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +- .../ozone/shell/volume/ListVolumeHandler.java | 9 +- .../ozone/debug/TestDBDefinitionFactory.java | 7 +- pom.xml | 74 +- 519 files changed, 11634 insertions(+), 9037 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
