This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from f386662 HDDS-6381. EC: Improve exception message in
ByteBufferEncodingState (#3143)
add 000902d HDDS-6325. Fix interface ClientProtocol methods typo
setThreadLocalS3Auth and clearThreadLocalS3Auth (#3093)
add f0f344d HDDS-6311. Fix number of keys displayed in Recon Overview.
(#3081)
add 87ff329 HDDS-6305: Add metrics - number of FSO bucket creates (#3077)
add 8c5fefe HDDS-6330. Remove unnecessary duplicate semicolons (#3097)
add ff0209d HDDS-6331. Remove toString in debug log parameters within
SCMCommonPlacementPolicy (#3098)
add 2c561d0 HDDS-6283. Change ContainerStateMachine ContainerOpExecutor
name (#3055)
add eb0f56b HDDS-6324. Do not trigger CI by reopening PR (#3092)
add c87b059 HDDS-6284. Add BlockDeletingService worker size config (#3056)
add 51f47fc HDDS-6314. ConcurrentModificationException getting
SCMContainerMetrics (#3101)
add 53fa44d HDDS-6290. operational-state and node-state options in
datanode list CLI not working correctly (#3105)
add 91cf777 HDDS-6262. ozone insight log stops working after OM DBUpdates
message (#3044)
add 07fa775 HDDS-6234. Repair containers affected by incorrect used bytes
and block count. (#3042)
add 788ff0f HDDS-6212. SCM Container DB bootstrap on Recon startup for
secure cluster (#3027)
add ad33ac6 HDDS-6292. Ensure immutable ContainerReplica set is returned
from ContainerStateManagerImpl (#3071)
add 8ad0796 HDDS-6329. New checkstyle: AvoidStarImport (#3096)
add 7f6e6fb HDDS-6350. Rename TestStandardOutputUtil to observe naming
conventions (#3114)
add 4b935c5 HDDS-6328. Add metrics - number of FSO bucket deletes (#3107)
add e848ab7 HDDS-6327. Upgrade acceptance test doesn't collect logs when
the test fails (#3106)
add 68c5ac5 HDDS-6351. [Docs] Update prefix-based FSO documentation.
(#3115)
add 239d889 HDDS-5267. Full Container Report can remove replicas added by
an Incremental Report (#2963)
add f0ee98a HDDS-6363. Update checkstyle version to 9.3 (#3123)
add 1c7ff7e HDDS-6221. Update versions covered by cross-compatibility
test (#3017)
add a195100 HDDS-5359. Incorrect BLOCKCOUNT and BYTESUSED in container
DB. (#3034)
add 5f4c31a HDDS-6349. IncompleteReadError on get MPU key from TDE bucket
(#3116)
add a53a3a3 HDDS-6235. Empty KeyValueContainers are replicated without
chunks directory. (#3052)
add 7310491 HDDS-5194. Provide kustomize descriptors for Ozone kubernetes
examples (#2221)
add 93631a1 HDDS-6334. Remove ContainerID to Proto to ContainerID
conversion in ContainerStateManagerImpl (#3110)
add 17d86f1 HDDS-6361. Modify docs build flow to replace image tags with
shortcodes. (#3122)
add da93105 HDDS-6369. PARTIAL_RENAME does not update OM DB (#3126)
add fbce851 HDDS-6374. Fix incorrect queueTime metrics of ReplicationTask
(#3135)
add ebbe9b5 HDDS-6337. [FSO] Disable recursive access check flag for
directories with no children. (#3134)
add 898ccbb HDDS-6376. Docs: Fix classpath for ofs and o3fs (#3138)
add 53d25b9 HDDS-6307. Improve processing and memory efficiency of full
container reports (#3085)
add 6e117f6 HDDS-6395. Intermittent failure in
TestReconScmHASnapshot.testScmHASnapshot (#3149)
add 2ed5994 HDDS-6268. Include audit log in acceptance test bundle (#3045)
add 5f029d0 HDDS-6317: Export ReconTaskStatus as Prometheus metrics
(#3088)
add ba31f8c HDDS-6333. Add a metric to record sequence number lag between
Recon and OM (#3100)
add e9722b7 HDDS-6401. Fix flaky
TestFilePerBlockStrategy.testWriteAndReadChunkMultipleTimes (#3152)
add d0cde3a HDDS-5712. make it configurable to trigger refresh datanode
usage info before start a new balance iteration (#2944)
add b83c1f9 HDDS-6244. ContainerBalancer metrics don't show updated
values in JMX (#3049)
add e716c18 HDDS-6321. Avoid refresh pipeline for key lookup in checkAcls
(#3103)
add fe6f060 HDDS-6370. Document connecting IDE to Ozone on Kubernetes
(#3132)
add 94598d1 HDDS-6420. Ratis and Standalone ReplicationConfig should
return a static instance (#3165)
add 5217134 HDDS-6360. Create stack dump on acceptance test failure
(#3154)
add b439802 HDDS-6417 - Fix console output message when container
balancer does not start (#3164)
add bf38696 HDDS-6423. Fix typo in TestTools.md (#3170)
add 2a1b6ba HDDS-6379. Not deducting the STANDALONE pipelines when
counting pipelines on each datanode to check the pipeline limit (#3158)
add 58207a2 HDDS-6153. Add simple integration test to the read-replicas
debug tool. (#3065)
add 1c58d79 HDDS-6415 - add Over-Utilized and Under-Utilized DN details
in debug log (#3171)
add 5587023 HDDS-6357. RenameKey request has memory leak (#3121)
add 2a776a3 HDDS-6306 - Fix picocli warnings in ozone container balancer
help (#3162)
add ea06d26 HDDS-6419. Provide better error message for malformed auth
header (#3167)
add 759ac57 HDDS-6407. Unwrap RemoteException in getClientProtocol (#3157)
add 933a19c HDDS-5656. Move old objects to delete table on overwriting
multipart objects (#2813)
add 1c414bf HDDS-6299. Remove Log4J2 from Ozone FS shaded jars (#3073)
add a720211 HDDS-6267. Recon is accessing mock classes. (#3084)
add 27aad3c HDDS-6391. ClientVersions and DatanodeVersions class to
define an enum with version and description (#3155)
add 7cfd846 HDDS-6251. EC: Smoketest for ozone admin datanode expects
exactly 3 nodes (#3182)
add b01138c HDDS-6392. Introduce OzoneManagerVersion enum (#3159)
add bb212de HDDS-6418. Datanode usage info uses wrong version number
(#3173)
add f826d12 HDDS-6431. Fix usedBytes for FSO bucket (#3178)
add 5a1babf HDDS-6439. Minor perf optimization in the version enums
(#3189)
add 4454e6b HDDS-6095. Skip slow integration tests, allow repeating known
flaky tests 5x in CI (#3148)
add 5f3f9fa HDDS-4171. Typo in MiniOzoneCluster: DEFAULT_PIPELIME_LIMIT
(#3191)
add c2a67ac HDDS-6450. [FSO] Fix correctedSpace for usedBytes in
commitKey (#3193)
add 74d92c8 HDDS-6213. Pluggable OzoneManager request handling hooks
(#3104)
add 909b0d2 HDDS-6079. Replace Freon RK with OCKG/OCKV in acceptance
tests (#2916)
add be2021a HDDS-6470. Fix TestOzoneManagerHAWithData#testOMRestart()
(#3213)
add ea296fe HDDS-6468. Set correct resource type for ACL checks in
BucketAcl and KeyAcl requests (#3209)
add 416439d HDDS-5867. Update quota when deleting open keys (#3206)
add 97d8ed7 HDDS-6432. Smoketest for read-replicas tool expects exactly 3
datanodes (#3187)
add 6408198 HDDS-6464. Turn off cross-check fail-fast for CI (#3208)
add ca24bdb HDDS-5228 Make OM FailOverProxyProvider work across threads
(#3160)
add d97687c HDDS-6087. Compute composite CRC file checksum using chunk
checksums from DataNodes. (#2937)
add 4da4cc1 HDDS-6477. Intermittent failure in
TestOzoneManagerHAMetadataOnly#testListVolumes (#3219)
add 61a313f HDDS-6228. Update config keys for open key cleanup service
(#3022)
add 042aa8b HDDS-3370. Cleanup old write-path of bucket in OM (#3183)
add 92995ec HDDS-6261. OM crashes when trying to overwrite a key during
upgrade downgrade testing (#3174)
add a8ed41e HDDS-6490. Replace deprecated --no-ansi option in acceptance
tests (#3224)
add a5602a8 HDDS-6480. Add missing CleanupTableInfo to OM responses
(#3220)
add 1baa652 HDDS-6488. Intermittent failure in ozone debug read-replicas
with one datanode STALE (#3223)
add 4ff6300 HDDS-6398. Add audit logs for DelegationToken (#3163)
add 2f2f525 HDDS-6472. Errors in TestOzoneContainer integration test
(#3216)
add cfb8bae HDDS-6393. Change the OzoneManager ServiceInfo message to
carry the int version from OzoneManagerVersions. (#3211)
add 9f716b8 HDDS-6482. Refactor Ratis Rpc and TLS config code. (#3222)
add 75f5501 HDDS-6481. Add metrics for S3 gateway (#3221)
new 9ec90f5 Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec-merge-master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 12 +-
dev-support/annotations/pom.xml | 114 +++++
.../RequestFeatureValidatorProcessor.java | 289 ++++++++++++
.../org/apache/ozone/annotations/package-info.java | 5 +
.../services/javax.annotation.processing.Processor | 19 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
hadoop-hdds/common/pom.xml | 13 +-
...DatanodeVersions.java => ComponentVersion.java} | 24 +-
.../org/apache/hadoop/hdds/DatanodeVersion.java | 71 +++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 1 -
.../hadoop/hdds/client/RatisReplicationConfig.java | 34 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/StandaloneReplicationConfig.java | 34 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 66 ++-
.../RequestTypeDependentRetryPolicyCreator.java | 6 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 21 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 11 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 75 +++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 32 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 72 +++
.../common/src/main/resources/ozone-default.xml | 35 +-
.../hdds/TestComponentVersionInvariants.java | 98 ++++
.../client/TestReplicationConfigValidator.java | 16 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 11 +-
.../hdds/scm/container/TestContainerInfo.java | 2 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 9 +-
.../ozone/container/ContainerTestHelper.java | 68 ---
.../TestDefaultUpgradeFinalizationExecutor.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +-
.../ozone/container/common/impl/ContainerData.java | 46 +-
.../ozone/container/common/impl/ContainerSet.java | 16 +-
.../common/interfaces/ContainerInspector.java | 72 +++
.../ozone/container/common/interfaces/Handler.java | 12 +-
.../common/report/ContainerReportPublisher.java | 5 +-
.../common/report/IncrementalReportSender.java | 30 ++
.../common/statemachine/DatanodeStateMachine.java | 6 +-
.../common/statemachine/StateContext.java | 131 +++---
.../CreatePipelineCommandHandler.java | 13 +-
.../states/datanode/RunningDatanodeState.java | 8 +-
.../states/endpoint/HeartbeatEndpointTask.java | 23 +-
.../server/ratis/ContainerStateMachine.java | 7 +-
.../transport/server/ratis/XceiverServerRatis.java | 18 +-
.../common/utils/ContainerInspectorUtil.java | 87 ++++
.../container/common/volume/AbstractFuture.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 76 ++-
.../container/keyvalue/KeyValueContainerData.java | 10 +-
.../KeyValueContainerMetadataInspector.java | 463 ++++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 95 ++--
.../container/keyvalue/TarContainerPacker.java | 55 ++-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 104 ++--
.../keyvalue/impl/ChunkManagerDispatcher.java | 18 +-
.../background/BlockDeletingService.java | 108 +++--
.../container/metadata/AbstractDatanodeStore.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +-
.../container/replication/MeasuredReplicator.java | 3 +-
.../commands/RefreshVolumeUsageCommand.java | 57 +++
.../hadoop/ozone/container/common/ScmTestMock.java | 7 +-
.../container/common/TestBlockDeletingService.java | 6 +-
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 47 --
.../container/common/impl/TestHddsDispatcher.java | 7 +-
.../common/statemachine/TestStateContext.java | 71 ++-
.../TestCreatePipelineCommandHandler.java | 36 +-
.../common/volume/TestStorageVolumeChecker.java | 12 +-
.../keyvalue/TestKeyValueBlockIterator.java | 10 +-
.../container/keyvalue/TestKeyValueContainer.java | 67 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 158 +------
...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++-----
.../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 19 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 -
.../keyvalue/impl/CommonChunkManagerTestCases.java | 1 -
.../keyvalue/impl/TestBlockManagerImpl.java | 57 +--
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../replication/TestMeasuredReplicator.java | 15 +
.../testutils/BlockDeletingServiceTestImpl.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
hadoop-hdds/docs/content/concept/Containers.md | 3 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 9 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 ++-
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/tools/TestTools.md | 4 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 4 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 +++
.../themes/ozonedoc/layouts/shortcodes/image.html | 2 +-
hadoop-hdds/framework/pom.xml | 8 +
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 37 ++
...lockLocationProtocolClientSideTranslatorPB.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 9 +-
.../scm/update/client/UpdateServiceConfig.java | 5 +-
.../authority/PKIProfiles/DefaultProfile.java | 32 +-
...ateClient.java => CommonCertificateClient.java} | 116 ++---
.../certificate/client/OMCertificateClient.java | 79 +---
.../client/ReconCertificateClient.java} | 38 +-
...va => FixedThreadPoolWithAffinityExecutor.java} | 70 ++-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 75 ---
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 50 +-
.../hdds/utils/db/cache/PartialTableCache.java | 56 +--
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../client/TestDefaultCertificateClient.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 35 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 77 ++-
hadoop-hdds/interface-client/pom.xml | 5 +
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 7 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 11 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 8 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +-
.../container/AbstractContainerReportHandler.java | 43 +-
.../hdds/scm/container/ContainerManager.java | 7 +
.../hdds/scm/container/ContainerManagerImpl.java | 78 +--
.../hdds/scm/container/ContainerReportHandler.java | 141 ++++--
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../scm/container/ContainerStateManagerImpl.java | 24 +-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../scm/container/balancer/ContainerBalancer.java | 133 ++++--
.../balancer/ContainerBalancerConfiguration.java | 23 +-
.../balancer/ContainerBalancerMetrics.java | 139 +++---
.../scm/container/states/ContainerStateMap.java | 27 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 37 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 8 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +-
...ffer.java => SCMHADBTransactionBufferStub.java} | 8 +-
...MockSCMHAManager.java => SCMHAManagerStub.java} | 24 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 4 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +-
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 6 +-
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 6 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 15 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 18 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 34 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 4 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 54 +--
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 6 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 3 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 28 ++
...inerLocationProtocolServerSideTranslatorPB.java | 9 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 2 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 12 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 12 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 16 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 51 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 14 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 7 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 21 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +-
.../container/TestCloseContainerEventHandler.java | 8 +-
.../scm/container/TestContainerManagerImpl.java | 26 +-
.../scm/container/TestContainerReportHandler.java | 34 +-
.../scm/container/TestContainerStateManager.java | 14 +-
.../TestIncrementalContainerReportHandler.java | 30 +-
.../hdds/scm/container/TestReplicationManager.java | 140 +++---
.../scm/container/TestUnknownContainerReport.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 84 ++--
.../states/TestContainerReplicaCount.java | 5 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 10 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 6 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 7 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 24 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 12 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 4 +-
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 129 ++---
.../scm/pipeline/TestPipelinePlacementPolicy.java | 158 ++++++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 62 +--
.../hdds/scm/pipeline/TestPipelineStateMap.java | 12 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 57 ++-
.../scm/pipeline/TestSimplePipelineProvider.java | 18 +-
.../pipeline/TestWritableECContainerProvider.java | 16 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 20 +-
.../TestOneReplicaPipelineSafeModeRule.java | 10 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 22 +-
.../scm/server/TestSCMBlockProtocolServer.java | 10 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +--
.../server/TestStorageContainerManagerStarter.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 14 +
hadoop-hdds/test-utils/pom.xml | 5 +
.../org/apache/ozone/test/GenericTestUtils.java | 18 +-
.../java/org/apache/ozone/test/tag/Flaky.java} | 36 +-
.../main/java/org/apache/ozone/test/tag/Slow.java} | 35 +-
.../org/apache/ozone/test/tag/package-info.java} | 23 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 8 +-
.../hdds/scm/cli/ContainerOperationClient.java | 9 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 6 +-
.../datanode/TestContainerBalancerSubCommand.java | 6 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 12 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../client/checksum/BaseFileChecksumHelper.java | 54 ++-
.../checksum/ReplicatedFileChecksumHelper.java | 44 +-
.../client/io/BlockOutputStreamEntryPool.java | 1 -
.../hadoop/ozone/client/io/ECBlockInputStream.java | 7 +-
.../client/io/MultipartCryptoKeyInputStream.java | 4 +
.../ozone/client/protocol/ClientProtocol.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 46 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 39 +-
.../hadoop/ozone/client/rpc/RpcClientTest.java | 279 +++++------
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 15 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 130 ++---
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 16 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 3 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 4 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 25 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 3 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 21 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 37 +-
.../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 18 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 24 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 5 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 5 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 5 +-
hadoop-ozone/csi/pom.xml | 4 +
hadoop-ozone/datanode/pom.xml | 1 +
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
.../dev-support/checks/_mvn_unit_report.sh | 24 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-topology/test.sh | 6 -
.../dist/src/main/compose/ozone/docker-config | 4 +
.../dist/src/main/compose/ozone/prometheus.yml | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +
.../src/main/compose/ozonesecure/docker-config | 4 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 98 +++-
.../dist/src/main/compose/upgrade/testlib.sh | 9 +-
.../non-rolling-upgrade/1.1.0-1.2.0/callback.sh | 6 +
.../dist/src/main/compose/xcompat/clients.yaml | 18 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +-
hadoop-ozone/dist/src/main/k8s/README.md | 68 +++
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/examples/getting-started/Flekszible | 2 +
.../examples/getting-started/config-configmap.yaml | 1 +
.../kustomization.yaml} | 30 +-
.../dist/src/main/k8s/examples/minikube/Flekszible | 2 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../Flekszible => minikube/kustomization.yaml} | 30 +-
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../examples/ozone-dev/kustomization.yaml} | 42 +-
.../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 +
.../ozone-ha/{Flekszible => kustomization.yaml} | 26 +-
.../dist/src/main/k8s/examples/ozone/Flekszible | 3 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../Flekszible => ozone/kustomization.yaml} | 26 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
.../main/smoketest/auditparser/auditparser.robot | 22 +-
.../dist/src/main/smoketest/basic/basic.robot | 9 +-
.../debug/ozone-debug-corrupt-block.robot | 49 ++
.../ozone-debug-dead-datanode.robot} | 35 +-
.../debug/ozone-debug-stale-datanode.robot | 48 ++
.../main/smoketest/debug/ozone-debug-tests.robot | 51 ++
.../src/main/smoketest/debug/ozone-debug.robot | 93 +++-
.../dist/src/main/smoketest/freon/generate.robot | 19 +-
.../dist/src/main/smoketest/freon/remove.robot | 21 +-
.../dist/src/main/smoketest/freon/validate.robot | 13 +-
.../dist/src/main/smoketest/omha/om-prepare.robot | 3 +-
.../dist/src/main/smoketest/ozone-lib/freon.robot | 65 +++
.../dist/src/main/smoketest/recon/recon-api.robot | 6 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 21 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 7 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++
.../dist/src/main/smoketest/spnego/web.robot | 4 -
.../dist/src/main/smoketest/upgrade/generate.robot | 5 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 25 +
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 26 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 4 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 23 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 7 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 19 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 5 +-
.../TestContainerStateManagerIntegration.java | 44 +-
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 28 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 29 +-
.../TestRatisPipelineCreateAndDestroy.java | 31 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 7 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 33 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 8 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 20 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 5 +-
...OutputUtil.java => StandardOutputTestBase.java} | 2 +-
.../hadoop/ozone/TestContainerOperations.java | 35 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 36 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 8 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +-
.../hadoop/ozone/TestStorageContainerManager.java | 5 +-
.../ozone/client/TestOzoneClientFactory.java | 75 +++
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 23 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 23 +-
.../rpc/TestCloseContainerHandlingByClient.java | 14 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../TestContainerStateMachineFailureOnRead.java | 21 +-
.../rpc/TestContainerStateMachineFailures.java | 13 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 14 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 8 +-
.../client/rpc/TestFailureHandlingByClient.java | 144 +++++-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 28 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 18 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 41 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 3 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 10 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 20 +-
.../ozone/client/rpc/TestWatchForCommit.java | 12 +-
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 4 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 2 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 8 +-
.../commandhandler/TestBlockDeletion.java | 21 +-
.../TestCloseContainerByPipeline.java | 24 +-
.../commandhandler/TestCloseContainerHandler.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
...ler.java => TestRefreshVolumeUsageHandler.java} | 121 +++--
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/ozoneimpl/TestOzoneContainer.java | 154 +++---
.../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +-
.../container/server/TestContainerServer.java | 23 +-
.../server/TestSecureContainerServer.java | 10 -
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 15 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 16 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 19 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +-
.../ozone/om/TestContainerReportWithKeys.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 36 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 -
.../apache/hadoop/ozone/om/TestObjectStore.java | 12 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 82 +++-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 7 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 5 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 4 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 25 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 2 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 40 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 62 +--
.../ozone/om/TestOzoneManagerHAWithFailover.java | 4 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 2 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 49 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 9 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 8 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 7 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 5 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 18 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 27 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +-
hadoop-ozone/interface-client/pom.xml | 4 +
.../src/main/proto/OmClientProtocol.proto | 3 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 6 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 6 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 3 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 3 +-
hadoop-ozone/ozone-manager/pom.xml | 5 +
.../org/apache/hadoop/ozone/om/BucketManager.java | 22 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 255 ----------
.../hadoop/ozone/om/DirectoryDeletingService.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 9 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 113 +++--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 20 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 29 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 22 +
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 20 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 15 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 15 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 4 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/file/OMFileRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 52 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 40 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 42 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 64 ++-
.../S3MultipartUploadCompleteRequestWithFSO.java | 12 +-
.../security/OMCancelDelegationTokenRequest.java | 42 +-
.../security/OMGetDelegationTokenRequest.java | 54 ++-
.../security/OMRenewDelegationTokenRequest.java | 51 +-
.../validation/RequestFeatureValidator.java | 99 ++++
.../request/validation/RequestProcessingPhase.java | 36 +-
.../om/request/validation/RequestValidations.java | 107 +++++
.../om/request/validation/ValidationCondition.java | 55 +++
.../om/request/validation/ValidationContext.java | 52 ++
.../om/request/validation/ValidatorRegistry.java | 201 ++++++++
.../ozone/om/request/validation/package-info.java | 62 +++
.../om/response/bucket/OMBucketCreateResponse.java | 3 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 3 +-
.../response/file/OMFileCreateResponseWithFSO.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 3 +-
.../key/OMAllocateBlockResponseWithFSO.java | 3 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 4 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 3 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysDeleteResponse.java | 4 +-
.../om/response/key/OMKeysRenameResponse.java | 10 +
.../om/response/key/OMOpenKeysDeleteResponse.java | 18 +-
.../multipart/S3MultipartUploadAbortResponse.java | 9 +-
.../S3MultipartUploadCommitPartResponse.java | 9 +-
.../S3MultipartUploadCompleteResponse.java | 36 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 9 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 25 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 11 +
.../ozone/om/ScmBlockLocationTestingClient.java | 2 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 243 +++++-----
.../hadoop/ozone/om/TestKeyDeletingService.java | 2 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 14 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 50 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../apache/hadoop/ozone/om/TestTrashService.java | 4 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 +
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +
.../bucket/TestOMBucketDeleteRequestWithFSO.java | 76 +++
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 97 +++-
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 174 ++++---
.../TestS3MultipartUploadCompleteRequest.java | 32 +-
.../security/TestOMGetDelegationTokenRequest.java | 10 +-
.../TestRequestFeatureValidatorProcessor.java | 524 +++++++++++++++++++++
.../request/validation/TestRequestValidations.java | 349 ++++++++++++++
.../request/validation/TestValidatorRegistry.java | 215 +++++++++
.../GeneralValidatorsForTesting.java | 190 ++++++++
.../ValidatorsForOnlyOldClientValidations.java | 43 ++
.../om/response/key/TestOMKeyDeleteResponse.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 61 ++-
.../s3/multipart/TestS3MultipartResponse.java | 19 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 17 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 2 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 2 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 4 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 +
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 3 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 5 +
hadoop-ozone/recon-codegen/pom.xml | 4 +
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 125 +++++
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 ++
.../ozone/recon/api/ClusterStateEndpoint.java | 15 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 ++++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 ++-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 20 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestOpenContainerCount.java | 16 +-
.../recon/fsck/TestContainerHealthStatus.java | 4 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +-
.../TestContainerHealthTaskRecordGenerator.java | 3 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 20 +-
.../ozone/recon/scm/TestReconContainerManager.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 12 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 15 +-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 4 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 4 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 9 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 24 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 13 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 12 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 34 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 9 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 320 +++++++++++++
.../hadoop/ozone/s3/metrics/package-info.java | 23 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 113 +++++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 4 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 3 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 5 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../freon/containergenerator/GeneratorOm.java | 5 +-
.../freon/containergenerator/GeneratorScm.java | 4 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 5 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +-
pom.xml | 85 +++-
580 files changed, 11454 insertions(+), 4601 deletions(-)
create mode 100644 dev-support/annotations/pom.xml
create mode 100644
dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
create mode 100644
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
rename .github/workflows/cancel-ci.yaml =>
dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
(61%)
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/{DatanodeVersions.java
=> ComponentVersion.java} (67%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersion.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerInspector.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/IncrementalReportSender.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerInspectorUtil.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java
copy
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/{TestKeyValueContainerCheck.java
=> TestKeyValueContainerIntegrityChecks.java} (52%)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
create mode 100644 hadoop-hdds/docs/dev-support/bin/make_images_responsive.py
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/{OMCertificateClient.java
=> CommonCertificateClient.java} (51%)
copy
hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java
=>
framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java}
(51%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{FixedThreadPoolExecutor.java
=> FixedThreadPoolWithAffinityExecutor.java} (59%)
delete mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/EpochEntry.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHADBTransactionBuffer.java
=> SCMHADBTransactionBufferStub.java} (92%)
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/{MockSCMHAManager.java
=> SCMHAManagerStub.java} (91%)
copy
hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java
=> test-utils/src/main/java/org/apache/ozone/test/tag/Flaky.java} (56%)
rename
hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ClientVersions.java
=> test-utils/src/main/java/org/apache/ozone/test/tag/Slow.java} (59%)
copy
hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java
=> test-utils/src/main/java/org/apache/ozone/test/tag/package-info.java} (62%)
create mode 100644 hadoop-ozone/dist/src/main/k8s/README.md
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone-ha/Flekszible =>
getting-started/kustomization.yaml} (71%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone-ha/Flekszible =>
minikube/kustomization.yaml} (71%)
copy hadoop-ozone/dist/src/main/{compose/ozone/prometheus.yml =>
k8s/examples/ozone-dev/kustomization.yaml} (56%)
copy hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/{Flekszible =>
kustomization.yaml} (71%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone-ha/Flekszible =>
ozone/kustomization.yaml} (71%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
copy hadoop-ozone/dist/src/main/smoketest/{freon/validate.robot =>
debug/ozone-debug-dead-datanode.robot} (50%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestStandardOutputUtil.java
=> StandardOutputTestBase.java} (98%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientFactory.java
copy
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{TestCloseContainerHandler.java
=> TestRefreshVolumeUsageHandler.java} (55%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/UpdateServiceConfig.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java
(54%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketDeleteRequestWithFSO.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconTaskStatusMetrics.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
rename
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java
=>
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/package-info.java
(62%)
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
