This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
from be5ffc29e2 HDDS-10471. [hsync] MockDatanodeStorage.writeChunk should
make a copy of byte string. (#6334)
add 039dea9129 HDDS-10229. Fixes for Grafana dashboards (#6120)
add 3d8365ffd2 HDDS-10296. Orphan blocks during overwrite of key. (#6180)
add 06399b5dcb HDDS-10226. Refactor OMRequestTestUtils.createOmKeyInfo
(#6184)
add e93e781ffc HDDS-10291. Set simple properties in MiniOzoneCluster via
Configuration (#6186)
add 5715aee571 HDDS-10328. Support cross realm Kerberos out of box. (#6192)
add 601fd413a0 HDDS-10322. Make VolumeArgs immutable (#6193)
add c1efa330d3 HDDS-8627. Recon - API for Count of deletePending
directories (#5037)
add 15b62de75f HDDS-10333. RocksDB logger not closed (#6200)
add 75df6c1b75 HDDS-9843. Ozone client high memory (heap) utilization
(#6153)
add d3e2e59c1c HDDS-10319. Also consider bucket layout deciding whether to
normalize path for listKeys (#6195)
add 2f2234c7b6 HDDS-10262. Encapsulate SnapshotCache inside
OmSnapshotManager (#6135)
add af25a48005 HDDS-10340. Skip ci on dashboard updates (#6203)
add 47ef84c59c HDDS-10318. Add OM client protocol metrics to Ozone -
ListKey Metrics dashboard (#6183)
add 7c79246836 HDDS-10256. Retry block allocation when SCM is in safe
mode. (#6189)
add c35e99f04b HDDS-10250. Use SnapshotId as key in SnapshotCache (#6139)
add ffd8221eba HDDS-10343. Remove dependency on jsr305 (#6208)
add cd0069113e HDDS-10218. Speed up TestSstFilteringService (#6196)
add 45c853c5eb HDDS-10325. Make BucketArgs immutable (#6205)
add c289c670b9 HDDS-10344. Schedule dependabot for weekend (#6209)
add 68662a757a HDDS-10347. Bump jacoco to 0.8.11 (#6214)
add bacb184fd5 HDDS-10356. Bump exec-maven-plugin to 3.1.1 (#6215)
add 7370676dcf HDDS-9680. Use md5 hash of multipart object part's content
as ETag (#5668)
add 3c4683e71b HDDS-9738. Display startTime, pipeline and container counts
for decommissioning datanode (#6185)
add 44adf80324 HDDS-10359. Recursively deleting volume with OBS bucket
shows error despite success (#6217)
add 38768527f3 HDDS-10368. Bump guice to 6.0.0 (#6212)
add 78fc781829 HDDS-10369. Set Times API doesn't work with linked buckets.
(#6220)
add 6194d422f3 HDDS-10301. Recon - Fold the pipeline info for a DN on
Datanode page. (#6198)
add 2348784bec HDDS-10288. Checksum to support direct buffers (#6162)
add cce2f969a8 HDDS-10339. Add S3 API level dashboard (#6202)
add e0f3ae1406 HDDS-10391. Bump joda-time to 2.12.7 (#6230)
add c6724537d7 HDDS-10387. Fix parameter number warning in KeyOutputStream
and related classes (#6225)
add 932a0ac93b HDDS-10342. Reduce code duplication in MiniOzoneCluster
builders (#6206)
add c8e6cabc8f HDDS-10345. No need to sort excluded datanodes during Ratis
pipeline creation (#6224)
add a1f839036a HDDS-10010. Support snapshot rename operation (#6006)
add db1561ac10 HDDS-10385. Memory leak for thread local usages in
OMClientRequest. (#6234)
add 652b9bcc0a HDDS-10401. Bump commons-compress to 1.26.0 (#6240)
add 3a01fea4ca HDDS-10406. Bump aws-java-sdk to 1.12.661 (#6249)
add 9c187df7df HDDS-10398. Remove deleted_blocks table in container schema
V2 and V3 definition (#6237)
add c9d3b23bb5 HDDS-10397. Restrict legacy bucket directory deletion
through sh command. (#6244)
add 5f6306dd1d HDDS-10395. Fix eTag compatibility issues for MPU (#6235)
add 45d420ab7c HDDS-10293. IllegalArgumentException: containerSize
Negative (#6178)
add f0b75b7e4e HDDS-10383. Introduce a Provider for client-side thread
resources passing (#6222)
add 6dfd7d46d1 HDDS-10408. NPE causes OM crash in Snapshot Purge request
(#6250)
add b537a6a710 HDDS-10396. Encapsulate fields in WithMetadata and
subclasses (#6238)
add d883d7df46 HDDS-10149. New JNI layer for RawSSTFileReader &
RawSSTFileReaderIterator (#6182)
add 284846f2c8 HDDS-10363. HDDS-9388 broke encryption. (#6219)
add 83cb2b7d1d HDDS-10320. Introduce factory to configure
MiniOzoneCluster's datanodes (#6246)
add 13b635c580 HDDS-10410. Avoid creating ChunkInfo. (#6258)
add babf85c762 HDDS-10403. CopyObject should set ETag based on the key
content (#6251)
add e9f11f0b5f HDDS-10404. Ozone admin reconfig command fails with
security enabled (#6253)
add 0bac7ef8e4 HDDS-10405. ozone admin has hard-coded info loglevel (#6254)
add c325315cae HDDS-10418. Bump commons-io to 2.15.1 (#6266)
add decacde8fc HDDS-10419. Bump maven-gpg-plugin to 3.1.0 (#6211)
add f62a8e33f3 HDDS-10420. Bump restrict-imports-enforcer-rule to 2.5.0
(#6264)
add 0cd6b3bf8a HDDS-10399. IndexOutOfBoundsException when shallow listing
empty directory in non-FSO bucket (#6259)
add dc9bd61914 HDDS-10365. Fix description for `ozone getconf
ozonemanagers` (#6263)
add df68290e72 HDDS-10214. Update supported versions in security policy up
to 1.4.0 (#6100)
add 84c6e4d861 HDDS-10384. RPC client Reusing thread resources. (#6270)
add 50d43e8ae4 HDDS-10394. Fix parameter number warning in om.helpers
(#6271)
add 1b48186a01 HDDS-10415. Remove duplicate HA MiniOzoneCluster factory
method (#6261)
add 9c2fb3aebd HDDS-10346. Make test cases in TestSstFilteringService
independent (#6247)
add 2d77fb4016 HDDS-10423. Datanode fails to start with invalid checksum
size setting (#6276)
add f6d455fd19 HDDS-10360. Make cleanupTest compatible with
enableFileSystemPath values in TestKeyManagerImpl class. (#6218)
add 083a45ee0b Revert "HDDS-10384. RPC client Reusing thread resources.
(#6270)" (#6277)
add 7939faf7d6 HDDS-815. Rename HDDS config keys prefixed with dfs. (#6274)
add 0e413c9833 HDDS-10428. OzoneClientConfig#validate does not get called
(#6282)
add 54548aa768 HDDS-10327. S3G does not work in a single-node deployment
(#6257)
add d3c5cceb2f Merge remote-tracking branch 'asf/master' into HDDS-7593
add 1d81c32de9 Merge remote-tracking branch 'asf/HDDS-7593' into HDDS-7593
add 9fb61ffa3b HDDS-10413. Recon - UnsupportedOperationException while
merging Incremental Container Reports. (#6260)
add 1e98ebb449 HDDS-10432. Hadoop FS client write(byte[], int, int) is
very slow in streaming (#6287)
add e0bf7b4f7b HDDS-10370. Recon - Handle the pre-existing missing empty
containers in clusters. (#6255)
add 8c4ab8e4cd HDDS-8683. Container balancer thread interrupt may not work
(#6179)
add f4406546ac HDDS-10416. Move HA-specific settings to
MiniOzoneHAClusterImpl.Builder (#6275)
add 4da5a64224 HDDS-10425. Increase OM transaction index for non-Ratis
based on existing Ratis transactionInfoTable (#6281)
add 543c9e79dd HDDS-9235. ReplicationManager metrics not collected after
restart. (#6280)
add 1830fe2055 HDDS-10367. Fix possible NPE in listKeysLight, listStatus,
listStatusLight (#6221)
add aa68aec220 HDDS-10324. Metadata are not updated when keys are
overwritten. (#6273)
add 4095ef1f48 HDDS-10278. Simplify tests using assertDoesNotThrow (#6291)
add 8fcd039cef HDDS-10437. Rename method to
getContainersPendingReplication (#6293)
add 01f8d62fbe HDDS-10331. Rename Java constants of ex-DFS config keys
(#6290)
add 1eeaa0baa2 HDDS-10144. Zero-Copy in replication (#6049)
add c3271b8bf4 HDDS-10144. (addendum) Zero-Copy in replication
add 083e9140a1 HDDS-10433. Add Prometheus scrape target for Datanodes in
Compose v2 (#6288)
add 804366a0f0 HDDS-6713. Avoid the need to cast to MiniOzoneHAClusterImpl
(#6295)
add 384103a2dd HDDS-10282. Fix pagination on the OM DB Insights page in
Recon (#6190)
add 0a5fc695b4 HDDS-7810. Support namespace summaries (du, dist & counts)
for OBJECT_STORE buckets. (#4245)
add 11fddc4191 HDDS-10041. Do not start the daemon inside the
OzoneManagerDoubleBuffer constructor. (#6300)
add 3a872b4a10 HDDS-10447. Extract helper methods from Ozone native ACL
unit tests (#6306)
add 2710129ce4 HDDS-10453. Bump httpclient to 4.5.14 (#6311)
add 61dbb0870a HDDS-10455. Bump protobuf-maven-plugin to 0.6.1 (#6313)
add 11c5eb86a4 HDDS-10456. Bump slf4j to 2.0.12 (#6312)
add b513cdcb54 HDDS-10439. Remove setConf from MiniOzoneCluster public
interface (#6320)
add a65991fa33 HDDS-10444. Reduce string concatenation in
ContainerImporter#importContainer (#6307)
add 650e77753b HDDS-10459. Bump snappy-java to 1.1.10.5 (#6324)
add a248ed1ea4 HDDS-10329. [Snapshot] Add unit-test for recreating
snapshots with deleted snapshot names. (#6298)
add ff1e4143ad HDDS-10430. Race condition around Pipeline#nodesInOrder
(#6316)
add f7a421b27b HDDS-10446. Refactor Node2ObjectsMap, Node2PipelineMap,
Node2ContainerMap (#6303)
add 80592135ed HDDS-10458. Mention `ozone admin datanode status
decommission` in docs (#6322)
add b69674c7f8 HDDS-10412. Prefix ACL check needs to resolve the bucket
link (#6268)
add f16b1affeb HDDS-10424. Improve error message for prefix without
trailing slash (#6328)
add 9a6ece27e7 HDDS-10457. Remove dependency commons-pool2 (#6317)
add 4243721dda HDDS-10450. Add GitHub actions labeler for the
reconciliation feature branch. (#6310)
add 87d8d6155c HDDS-10467. Reduce metrics visibility (#6332)
add 2f05353940 HDDS-10384. RPC client reusing thread resources. (#6326)
add 418528adea HDDS-10460. Refine audit logging for bucket property update
operations (#6329)
add 309e45996c HDDS-10472. Audit log should include EC replication config
(#6338)
add 140c5deffb HDDS-9343. Shift sortDatanodes logic to OM (#5391)
add a145dd5bfd HDDS-9343. (addendum) Shift sortDatanodes logic to OM
(#5391)
add 7c8160fe2c HDDS-10482. OMRequestTestUtils.createOmKeyInfo should set
key modification time (#6343)
add 3cadb248f5 Merge remote-tracking branch 'asf/master' into HDDS-7593
No new revisions were added by this update.
Summary of changes:
.github/dependabot.yml | 2 +
.github/{dependabot.yml => labeler.yml} | 33 +-
.github/{dependabot.yml => workflows/label-pr.yml} | 41 +-
SECURITY.md | 9 +-
dev-support/ci/selective_ci_checks.bats | 12 +
dev-support/ci/selective_ci_checks.sh | 2 +
.../hadoop/hdds/scm/ContainerClientMetrics.java | 15 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 17 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hdds/scm/storage/AbstractCommitWatcher.java | 2 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 9 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 1 +
.../hadoop/hdds/scm/storage/CommitWatcher.java | 23 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 7 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 20 +-
.../ozone/client/io/ByteBufferOutputStream.java | 5 +
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 38 +-
.../storage/TestBlockOutputStreamCorrectness.java | 5 +-
hadoop-hdds/common/pom.xml | 10 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../hadoop/hdds/client/ECReplicationConfig.java | 8 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 63 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 25 +
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 4 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 112 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 2 +
.../container/common/helpers/AllocatedBlock.java | 14 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 14 +
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 159 ++-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 20 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 20 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 15 +
.../hadoop/hdds/scm/net/NodeSchemaManager.java | 8 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 120 +-
.../protocol/StorageContainerLocationProtocol.java | 2 +
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 9 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 213 ++--
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 5 +
.../ozone/common/ChecksumByteBufferImpl.java | 29 +
.../apache/hadoop/ozone/common/ChunkBuffer.java | 10 +-
.../common/ChunkBufferImplWithByteBuffer.java | 14 +
.../ozone/common/IncrementalChunkBuffer.java | 19 +-
.../hadoop/ozone/grpc/metrics/GrpcMetrics.java | 4 +-
.../common/src/main/resources/ozone-default.xml | 113 +-
.../ozone/common/TestChecksumByteBuffer.java | 19 +
.../hadoop/ozone/common/TestChunkBuffer.java | 23 +-
.../org/apache/hadoop/hdds/conf/ConfigType.java | 6 +-
.../ozone/HddsDatanodeClientProtocolServer.java | 11 +-
.../apache/hadoop/ozone/HddsPolicyProvider.java | 43 +-
.../common/impl/ContainerLayoutVersion.java | 20 +-
.../common/transport/server/XceiverServerGrpc.java | 8 +-
.../common/transport/server/ratis/CSMMetrics.java | 15 +-
.../server/ratis/ContainerStateMachine.java | 12 +-
.../transport/server/ratis/XceiverServerRatis.java | 108 +-
.../ECReconstructionCoordinator.java | 47 +-
.../container/keyvalue/KeyValueContainerCheck.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 4 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 51 +-
.../keyvalue/impl/ChunkManagerFactory.java | 4 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 3 +-
.../metadata/AbstractDatanodeDBDefinition.java | 4 -
.../container/metadata/AbstractDatanodeStore.java | 10 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 1 -
.../metadata/DatanodeSchemaThreeDBDefinition.java | 18 -
.../metadata/DatanodeSchemaTwoDBDefinition.java | 17 -
.../metadata/DatanodeStoreSchemaOneImpl.java | 8 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 5 -
.../container/replication/ContainerImporter.java | 13 +-
.../replication/GrpcReplicationService.java | 93 +-
.../container/replication/MeasuredReplicator.java | 22 +-
.../container/replication/ReplicationServer.java | 36 +-
.../replication/SendContainerRequestHandler.java | 14 +-
.../ozone/container/common/ContainerTestUtils.java | 2 +-
.../ozone/container/common/SCMTestUtils.java | 4 +-
.../container/common/TestDatanodeStateMachine.java | 12 +-
.../statemachine/TestDatanodeConfiguration.java | 2 +-
.../common/volume/TestPeriodicVolumeChecker.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
.../common/volume/TestVolumeSetDiskChecks.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 5 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 2 +-
.../keyvalue/impl/TestFilePerChunkStrategy.java | 4 +-
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../replication/TestGrpcReplicationService.java | 178 ++-
.../TestGrpcReplicationServiceWithZeroCopy.java} | 34 +-
.../replication/TestReplicationSupervisor.java | 16 +-
.../TestSendContainerRequestHandler.java | 2 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 4 +-
hadoop-hdds/docs/content/feature/Decommission.md | 8 +
.../docs/content/feature/Decommission.zh.md | 8 +
.../docs/content/feature/Reconfigurability.md | 27 +-
.../docs/content/feature/Reconfigurability.zh.md | 27 +-
.../docs/content/feature/S3-Tenant-Commands.md | 2 +-
.../content/feature/Streaming-Write-Pipeline.md | 4 +-
.../ReconfigureProtocolClientSideTranslatorPB.java | 44 +-
...lPB.java => ReconfigureProtocolDatanodePB.java} | 11 +-
...rotocolPB.java => ReconfigureProtocolOmPB.java} | 11 +-
.../hdds/protocolPB/ReconfigureProtocolPB.java | 2 +-
.../ReconfigureProtocolServerSideTranslatorPB.java | 2 +-
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 127 ++
.../hadoop/hdds/scm/client/package-info.java} | 23 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 8 +
...lockLocationProtocolClientSideTranslatorPB.java | 43 +
...inerLocationProtocolClientSideTranslatorPB.java | 11 +
.../hdds/server/events/EventWatcherMetrics.java | 14 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 6 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 8 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 8 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 8 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 12 +
hadoop-hdds/hadoop-dependency-server/pom.xml | 12 +
.../src/main/proto/ScmAdminProtocol.proto | 11 +
hadoop-hdds/interface-client/pom.xml | 5 -
.../interface-client/src/main/proto/hdds.proto | 24 +
.../src/main/proto/ScmServerProtocol.proto | 11 +-
.../db/managed/JniLibNamePropertyWriter.java} | 40 +-
.../hdds/utils/db/managed/ManagedDBOptions.java | 14 +
.../{ManagedDBOptions.java => ManagedLogger.java} | 32 +-
hadoop-hdds/rocks-native/pom.xml | 205 +---
hadoop-hdds/rocks-native/src/CMakeLists.txt | 36 +-
.../apache/hadoop/hdds/utils/NativeConstants.java | 3 +-
.../db/managed/ManagedRawSSTFileIterator.java | 118 ++
.../utils/db/managed/ManagedRawSSTFileReader.java | 78 ++
.../utils/db/managed/ManagedSSTDumpIterator.java | 314 -----
.../hdds/utils/db/managed/ManagedSSTDumpTool.java | 99 --
.../hdds/utils/db/managed/PipeInputStream.java | 88 --
.../src/main/native/ManagedRawSSTFileIterator.cpp | 90 ++
.../src/main/native/ManagedRawSSTFileReader.cpp | 65 ++
hadoop-hdds/rocks-native/src/main/native/Pipe.h | 55 -
.../src/main/native/PipeInputStream.cpp | 48 -
.../rocks-native/src/main/native/SSTDumpTool.cpp | 48 -
.../src/main/native/cplusplus_to_java_convert.h | 2 +-
.../src/main/patches/rocks-native.patch | 1085 +++++++++---------
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 33 +-
.../db/managed/TestManagedRawSSTFileIterator.java | 143 +++
.../db/managed/TestManagedSSTDumpIterator.java | 282 -----
.../src/test/resources/auditlog.properties | 0
.../src/test/resources/log4j.properties | 0
.../ozone/rocksdb/util/SstFileSetReader.java | 209 ++--
.../ozone/rocksdb/util/TestSstFileSetReader.java | 62 +-
.../server-scm/dev-support/findbugsExcludeFile.xml | 5 +
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 220 ++++
.../scm/container/balancer/ContainerBalancer.java | 7 +-
.../container/replication/ReplicationManager.java | 1 +
.../replication/ReplicationManagerMetrics.java | 13 +-
.../hadoop/hdds/scm/node/DatanodeAdminMonitor.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 3 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 1 -
.../hdds/scm/node/NodeDecommissionManager.java | 4 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 1 -
.../hdds/scm/node/states/Node2ContainerMap.java | 92 --
.../hdds/scm/node/states/Node2PipelineMap.java | 28 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 -
...lockLocationProtocolServerSideTranslatorPB.java | 15 +
...inerLocationProtocolServerSideTranslatorPB.java | 12 +
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 12 +-
.../hdds/scm/security/RootCARotationMetrics.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 15 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 9 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 14 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 7 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 1 -
.../hdds/scm/container/Node2ContainerMap.java} | 101 +-
.../TestNode2ContainerMap.java | 6 +-
.../TestContainerReplicaPendingOps.java | 8 +
.../replication/TestECUnderReplicationHandler.java | 8 +
.../replication/TestReplicationManager.java | 8 +
.../hdds/scm/node/TestContainerPlacement.java | 2 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 8 +-
.../hadoop/hdds/scm/node/TestFetchMetrics.java | 46 +
.../ozone/container/common/TestEndPoint.java | 18 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 12 -
.../hdds/scm/cli/ContainerOperationClient.java | 9 +-
.../scm/cli/ReplicationManagerStartSubcommand.java | 7 +-
.../cli/ReplicationManagerStatusSubcommand.java | 9 +-
.../scm/cli/ReplicationManagerStopSubcommand.java | 9 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 11 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 7 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 19 +-
.../scm/cli/cert/CleanExpiredCertsSubcommand.java | 9 +-
.../hadoop/hdds/scm/cli/cert/InfoSubcommand.java | 16 +-
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 11 +-
.../hdds/scm/cli/cert/ScmCertSubcommand.java | 21 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 7 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 35 +-
.../hdds/scm/cli/container/ListSubcommand.java | 7 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 45 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 85 +-
.../cli/container/upgrade/TestUpgradeManager.java | 25 +-
.../datanode/TestDecommissionStatusSubCommand.java | 83 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 94 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 15 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 4 +-
.../client/OzoneMultipartUploadPartListParts.java | 19 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 2 +-
.../org/apache/hadoop/ozone/client/VolumeArgs.java | 47 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 81 +-
.../client/io/BlockOutputStreamEntryPool.java | 74 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 104 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 48 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 136 +--
.../hadoop/ozone/client/io/KeyOutputStream.java | 84 +-
.../ozone/client/protocol/ClientProtocol.java | 13 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 89 +-
.../client/io/TestECBlockOutputStreamEntry.java | 16 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 16 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 14 +-
.../hadoop/ozone/om/helpers/BasicOmKeyInfo.java | 67 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 165 ++-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 121 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 40 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 71 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 121 +-
.../helpers/OmMultipartCommitUploadPartInfo.java | 11 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 73 +-
.../om/helpers/OmMultipartUploadCompleteList.java | 5 +-
.../om/helpers/OmMultipartUploadListParts.java | 3 +-
.../apache/hadoop/ozone/om/helpers/OmPartInfo.java | 31 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 22 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 118 +-
.../hadoop/ozone/om/helpers/WithMetadata.java | 9 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 45 +-
.../ozone/om/helpers/WithParentObjectId.java | 10 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 15 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 91 +-
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 10 +
hadoop-ozone/csi/pom.xml | 14 +
hadoop-ozone/dev-support/checks/native.sh | 17 +-
.../intellij/runConfigurations/Datanode2-ha.xml | 2 +-
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3-ha.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
.../grafana/dashboards/Ozone - JVM Metrics.json | 357 ++----
.../dashboards/Ozone - ListKey Metrics.json | 196 ++--
.../grafana/dashboards/Ozone - Object Metrics.json | 1 -
.../grafana/dashboards/Ozone - S3 Dashboard.json | 1209 ++++++++++++++++++++
.../dist/src/main/compose/ozone-ha/docker-config | 2 +-
.../dist/src/main/compose/ozone/docker-config | 2 +-
.../dist/src/main/compose/ozone/prometheus.yml | 24 +-
.../src/main/compose/ozonesecure-ha/docker-config | 3 +-
.../main/compose/ozonesecure-ha/test-leadership.sh | 2 +-
.../src/main/compose/ozonesecure/docker-config | 3 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +-
.../LICENSE-com.google.code.findbugs-jsr305.txt | 8 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 4 +-
.../src/main/smoketest/admincli/reconfigure.robot | 38 +
.../dist/src/main/smoketest/basic/links.robot | 15 +
.../src/main/smoketest/s3/MultipartUpload.robot | 33 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 14 +
hadoop-ozone/dist/src/shell/ozone/ozone | 1 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 26 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 4 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 68 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 25 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 3 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 4 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 7 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 14 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 85 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 25 +-
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 2 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 4 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../hdds/scm/TestSCMInstallSnapshotWithHA.java | 2 +-
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 9 +-
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 5 +-
.../hdds/scm/TestStorageContainerManager.java | 18 +-
.../hdds/scm/TestStorageContainerManagerHA.java | 2 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 2 +-
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 9 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 9 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 5 +-
.../TestRatisPipelineCreateAndDestroy.java | 8 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 11 +-
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 9 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 31 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 18 +-
.../hadoop/hdds/utils/ClusterContainersUtil.java | 147 +++
.../db/managed/TestRocksObjectLeakDetector.java | 3 -
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 194 +---
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 281 ++---
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 124 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 6 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 5 +-
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 5 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 4 +
.../ozone/TestGetClusterTreeInformation.java | 87 ++
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 24 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 92 --
.../hadoop/ozone/TestMultipartObjectGet.java | 9 +-
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 187 +++
.../hadoop/ozone/TestSecureOzoneCluster.java | 33 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 167 +++
.../ozone/client/TestOzoneClientFactory.java | 3 +-
.../client/rpc/AbstractTestECKeyOutputStream.java | 7 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 3 -
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../client/rpc/TestBlockDataStreamOutput.java | 1 -
.../ozone/client/rpc/TestBlockOutputStream.java | 1 -
.../rpc/TestContainerReplicationEndToEnd.java | 4 +-
.../client/rpc/TestContainerStateMachine.java | 5 +-
.../TestContainerStateMachineFailureOnRead.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 14 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 5 +-
.../rpc/TestContainerStateMachineStream.java | 12 +-
.../client/rpc/TestDeleteWithInAdequateDN.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 5 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 4 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 4 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 3 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 99 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 176 +--
...estOzoneClientRetriesOnExceptionFlushDelay.java | 1 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 238 ++--
.../client/rpc/TestOzoneRpcClientWithRatis.java | 10 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 5 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 5 +-
.../commandhandler/TestBlockDeletion.java | 1 -
.../TestCloseContainerByPipeline.java | 2 +-
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../metrics/TestDatanodeQueueMetrics.java | 8 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 4 +-
.../container/server/TestContainerServer.java | 10 +-
.../server/TestSecureContainerServer.java | 10 +-
.../org/apache/hadoop/ozone/debug/TestLDBCli.java | 5 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 6 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 1 -
.../TestDatanodeHddsVolumeFailureToleration.java | 5 +-
.../hadoop/ozone/freon/TestDataValidate.java | 5 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 4 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 5 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 41 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 13 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 81 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 3 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 114 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 27 +
.../org/apache/hadoop/ozone/om/TestListStatus.java | 99 +-
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 11 +-
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 46 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 11 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 11 +-
.../ozone/om/TestOmContainerLocationCache.java | 11 +-
.../hadoop/ozone/om/TestOmSnapshotDisabled.java | 8 +-
.../ozone/om/TestOmSnapshotDisabledRestart.java | 5 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 20 +-
.../ozone/om/TestOzoneManagerHASnapshot.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 86 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 10 +-
.../om/TestOzoneManagerListVolumesSecure.java | 3 +
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 16 +-
.../ozone/om/TestSnapshotBackgroundServices.java | 18 +-
.../ozone/om/TestSnapshotDeletingService.java | 46 +-
.../om/TestSnapshotDirectoryCleaningService.java | 25 +-
.../om/multitenant/TestMultiTenantVolume.java | 5 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 67 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 -
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 11 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 2 +-
.../om/snapshot/TestOzoneSnapshotRestore.java | 9 +-
.../ozone/parser/TestOzoneHARatisLogParser.java | 2 +-
.../TestReconInsightsForDeletedDirectories.java | 28 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 25 +-
.../ozone/recon/TestReconWithOzoneManager.java | 10 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 10 +-
.../ozone/shell/TestDeletedBlocksTxnShell.java | 2 +-
.../shell/TestOzoneContainerUpgradeShell.java | 6 -
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 1 -
.../hadoop/ozone/shell/TestOzoneShellHA.java | 144 ++-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 14 +-
.../hadoop/ozone/shell/TestReconfigShell.java | 14 +-
.../apache/hadoop/ozone/shell/TestScmAdminHA.java | 4 +-
.../ozone/shell/TestTransferLeadershipShell.java | 2 +-
.../src/test/resources/ozone-site.xml | 22 +-
hadoop-ozone/interface-client/pom.xml | 6 +
.../src/main/proto/OmClientProtocol.proto | 24 +-
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 32 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 3 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 125 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 26 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 6 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 19 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 138 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 108 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 59 +-
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 11 +
.../hadoop/ozone/om/SnapshotChainManager.java | 8 +
.../hadoop/ozone/om/SstFilteringService.java | 18 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 12 +-
.../OzoneManagerDoubleBufferMetrics.java | 10 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 13 +-
.../ozone/om/ratis/metrics/package-info.java | 21 -
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMClientRequest.java | 14 +-
.../hadoop/ozone/om/request/RequestAuditor.java | 16 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 5 +-
.../request/file/OMFileCreateRequestWithFSO.java | 1 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 8 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 8 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 25 +-
.../ozone/om/request/key/OMKeySetTimesRequest.java | 4 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 86 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 29 +-
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 21 +-
.../key/acl/prefix/OMPrefixSetAclRequest.java | 21 +-
.../S3MultipartUploadCommitPartRequest.java | 10 +-
.../S3MultipartUploadCompleteRequest.java | 85 +-
.../request/snapshot/OMSnapshotPurgeRequest.java | 14 +-
.../request/snapshot/OMSnapshotRenameRequest.java | 230 ++++
.../key/OMDirectoriesPurgeResponseWithFSO.java | 12 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 12 +-
.../S3MultipartUploadCompleteResponse.java | 4 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 3 +-
.../OMSnapshotMoveDeletedKeysResponse.java | 21 +-
.../snapshot/OMSnapshotRenameResponse.java | 67 ++
.../ozone/om/service/DirectoryDeletingService.java | 10 +-
.../ozone/om/service/KeyDeletingService.java | 32 +-
.../ozone/om/service/SnapshotDeletingService.java | 23 +-
.../service/SnapshotDirectoryCleaningService.java | 32 +-
.../hadoop/ozone/om/snapshot/ReferenceCounted.java | 6 +-
.../hadoop/ozone/om/snapshot/SnapshotCache.java | 101 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 83 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 5 +
...OzoneManagerProtocolServerSideTranslatorPB.java | 12 +-
.../protocolPB/OzoneManagerRequestHandler.java | 27 +-
.../hadoop/ozone/protocolPB/RequestHandler.java | 39 +-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 5 +
.../ozone/om/ScmBlockLocationTestingClient.java | 11 +
.../hadoop/ozone/om/TestAuthorizerLockImpl.java | 9 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 135 +--
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 43 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 19 +-
.../hadoop/ozone/om/TestSstFilteringService.java | 126 +-
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 4 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 4 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 14 +-
.../ozone/om/request/OMRequestTestUtils.java | 314 ++---
.../request/bucket/TestOMBucketDeleteRequest.java | 13 +-
.../request/file/TestOMDirectoryCreateRequest.java | 11 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 19 +-
.../om/request/file/TestOMFileCreateRequest.java | 93 +-
.../file/TestOMFileCreateRequestWithFSO.java | 35 +-
.../om/request/file/TestOMRecoverLeaseRequest.java | 15 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 8 +-
.../key/TestOMAllocateBlockRequestWithFSO.java | 14 +-
.../TestOMDirectoriesPurgeRequestAndResponse.java | 2 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 6 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 23 +-
.../om/request/key/TestOMKeyCommitRequest.java | 66 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 17 +-
.../om/request/key/TestOMKeyCreateRequest.java | 157 ++-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 17 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 4 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 24 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 14 +-
.../om/request/key/TestOMKeyRenameRequest.java | 2 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 13 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 11 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 6 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 16 +-
.../om/request/key/TestOMKeysRenameRequest.java | 7 +-
.../om/request/key/TestOMPrefixAclRequest.java | 10 +-
.../om/request/key/TestOMSetTimesRequest.java | 2 +-
.../request/key/TestOMSetTimesRequestWithFSO.java | 15 +-
.../TestS3ExpiredMultipartUploadsAbortRequest.java | 19 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +-
.../TestS3MultipartUploadCommitPartRequest.java | 7 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 17 +-
.../TestS3MultipartUploadCompleteRequest.java | 22 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 17 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 12 +-
.../snapshot/TestOMSnapshotDeleteRequest.java | 3 -
.../TestOMSnapshotPurgeRequestAndResponse.java | 3 +-
...quest.java => TestOMSnapshotRenameRequest.java} | 301 +++--
.../file/TestOMDirectoryCreateResponse.java | 4 +-
.../file/TestOMFileCreateResponseWithFSO.java | 11 +-
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 16 +-
.../om/response/key/TestOMKeyCommitResponse.java | 4 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 26 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 15 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 6 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 26 +-
.../om/response/key/TestOMKeyRenameResponse.java | 6 +-
.../key/TestOMKeyRenameResponseWithFSO.java | 34 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 21 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 4 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 13 +-
.../om/response/key/TestOMKeysRenameResponse.java | 5 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 2 +-
.../key/acl/prefix/TestOMPrefixAclResponse.java | 23 +-
...TestS3ExpiredMultipartUploadsAbortResponse.java | 14 +-
.../s3/multipart/TestS3MultipartResponse.java | 2 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 56 +-
.../volume/TestOMVolumeDeleteResponse.java | 15 +-
.../om/service/TestDirectoryDeletingService.java | 15 +-
.../ozone/om/service/TestKeyDeletingService.java | 27 +-
.../service/TestMultipartUploadCleanupService.java | 4 +
.../om/service/TestOpenKeyCleanupService.java | 4 +
.../ozone/om/service/TestQuotaRepairTask.java | 21 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 122 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 125 +-
.../ozone/security/acl/OzoneNativeAclTestUtil.java | 170 +++
.../security/acl/TestOzoneNativeAuthorizer.java | 36 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 69 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 14 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 6 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 14 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 6 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 2 +
hadoop-ozone/ozonefs-shaded/pom.xml | 1 -
.../recon/schema/ContainerSchemaDefinition.java | 3 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 3 +
.../ozone/recon/api/OMDBInsightEndpoint.java | 31 +
.../ozone/recon/api/handlers/BucketHandler.java | 4 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 2 +-
.../recon/api/handlers/LegacyBucketHandler.java | 2 +-
.../ozone/recon/api/handlers/OBSBucketHandler.java | 268 +++++
.../api/types/UnhealthyContainersResponse.java | 13 +
.../ozone/recon/fsck/ContainerHealthTask.java | 47 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 30 +-
.../persistence/ContainerHealthSchemaManager.java | 11 +
.../ozone/recon/tasks/ContainerSizeCountTask.java | 79 +-
.../recon/tasks/DeletedKeysInsightHandler.java | 147 +++
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 41 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 236 ++++
.../hadoop/ozone/recon/tasks/OmTableHandler.java | 131 +++
.../ozone/recon/tasks/OmTableInsightTask.java | 321 ++----
.../ozone/recon/tasks/OpenKeysInsightHandler.java | 163 +++
.../webapps/recon/ozone-recon-web/api/db.json | 216 +---
.../webapps/recon/ozone-recon-web/api/routes.json | 33 +-
.../src/views/datanodes/datanodes.tsx | 41 +-
.../ozone-recon-web/src/views/insights/om/om.less | 17 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 384 ++-----
.../ozone/recon/OMMetadataManagerTestUtils.java | 18 +-
.../apache/hadoop/ozone/recon/TestReconUtils.java | 25 +
.../hadoop/ozone/recon/api/TestEndpoints.java | 24 +-
.../recon/api/TestNSSummaryEndpointWithOBS.java | 1127 ++++++++++++++++++
.../ozone/recon/api/TestOpenContainerCount.java | 26 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 130 ++-
...TestReconIncrementalContainerReportHandler.java | 27 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 16 +-
.../recon/tasks/TestContainerSizeCountTask.java | 74 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 4 +-
...maryTask.java => TestNSSummaryTaskWithOBS.java} | 428 ++++---
.../ozone/recon/tasks/TestOmTableInsightTask.java | 448 ++++++--
hadoop-ozone/s3gateway/pom.xml | 6 +
.../hadoop/ozone/s3/commontypes/KeyMetadata.java | 4 +-
.../endpoint/CompleteMultipartUploadRequest.java | 10 +-
.../endpoint/CompleteMultipartUploadResponse.java | 3 +-
.../ozone/s3/endpoint/CopyObjectResponse.java | 3 +-
.../hadoop/ozone/s3/endpoint/CopyPartResult.java | 3 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 3 +-
.../ozone/s3/endpoint/ListPartsResponse.java | 3 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 50 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 30 +-
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 4 +
.../hadoop/ozone/client/ClientProtocolStub.java | 7 +
.../hadoop/ozone/client/ObjectStoreStub.java | 3 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 34 +-
.../ozone/client/OzoneDataStreamOutputStub.java | 4 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 12 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 6 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 6 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 13 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 27 +
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 10 +-
.../s3/endpoint/TestPartUploadWithStream.java | 13 +-
.../ozone/s3/endpoint/TestUploadWithStream.java | 2 +-
.../reconfig/AbstractReconfigureSubCommand.java | 5 +-
.../ozone/admin/reconfig/ReconfigureCommands.java | 10 +
.../reconfig/ReconfigurePropertiesSubcommand.java | 5 +-
.../admin/reconfig/ReconfigureStartSubcommand.java | 5 +-
.../reconfig/ReconfigureStatusSubcommand.java | 5 +-
.../admin/reconfig/ReconfigureSubCommandUtil.java | 10 +-
.../ozone/conf/OzoneManagersCommandHandler.java | 3 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 3 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 8 +
.../shell/snapshot/RenameSnapshotHandler.java | 64 ++
.../ozone/shell/snapshot/SnapshotCommands.java | 3 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 +
pom.xml | 68 +-
599 files changed, 14635 insertions(+), 9074 deletions(-)
copy .github/{dependabot.yml => labeler.yml} (52%)
copy .github/{dependabot.yml => workflows/label-pr.yml} (52%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java
=>
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
(55%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPolicyProvider.java
=>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsPolicyProvider.java
(50%)
rename hadoop-hdds/{rocks-native/src/main/native/Pipe.cpp =>
container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationServiceWithZeroCopy.java}
(65%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/{ReconfigureProtocolPB.java
=> ReconfigureProtocolDatanodePB.java} (73%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/{ReconfigureProtocolPB.java
=> ReconfigureProtocolOmPB.java} (73%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmTopologyClient.java
copy
hadoop-hdds/{rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java
=>
framework/src/main/java/org/apache/hadoop/hdds/scm/client/package-info.java}
(72%)
copy
hadoop-hdds/{tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ReplicationManagerStartSubcommand.java
=>
managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java}
(51%)
copy
hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/{ManagedDBOptions.java
=> ManagedLogger.java} (62%)
create mode 100644
hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileIterator.java
create mode 100644
hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java
delete mode 100644
hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java
delete mode 100644
hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java
delete mode 100644
hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java
create mode 100644
hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp
create mode 100644
hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp
delete mode 100644 hadoop-hdds/rocks-native/src/main/native/Pipe.h
delete mode 100644 hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp
delete mode 100644 hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp
create mode 100644
hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java
delete mode 100644
hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedSSTDumpIterator.java
copy hadoop-hdds/{framework =>
rocks-native}/src/test/resources/auditlog.properties (100%)
copy hadoop-hdds/{client => rocks-native}/src/test/resources/log4j.properties
(100%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/FetchMetrics.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
rename
hadoop-hdds/server-scm/src/{main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
=> test/java/org/apache/hadoop/hdds/scm/container/Node2ContainerMap.java} (63%)
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/{node/states =>
container}/TestNode2ContainerMap.java (99%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestFetchMetrics.java
create mode 100644
hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - S3
Dashboard.json
delete mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.google.code.findbugs-jsr305.txt
create mode 100644
hadoop-ozone/dist/src/main/smoketest/admincli/reconfigure.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/utils/ClusterContainersUtil.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestGetClusterTreeInformation.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOMSortDatanodes.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/{metrics
=> }/OzoneManagerDoubleBufferMetrics.java (95%)
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/package-info.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotRenameRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotRenameResponse.java
copy
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/{TestOMSnapshotCreateRequest.java
=> TestOMSnapshotRenameRequest.java} (52%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAclTestUtil.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/OBSBucketHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithOBS.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBS.java
copy
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/{TestNSSummaryTask.java
=> TestNSSummaryTaskWithOBS.java} (57%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/RenameSnapshotHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
