This is an automated email from the ASF dual-hosted git repository.
duong pushed a change to branch HDDS-7733-Symmetric-Tokens
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit 783c90f72e HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
omit 124345cfe7 HDDS-8003. E2E integration test cases for block tokens
(#4547)
omit 84e260196f HDDS-7831. Use symmetric secret key to sign and verify
token (#4417)
omit 3f6ef59e16 HDDS-7830. SCM API for OM and Datanode to get secret keys
(#4345)
omit 89487e7fcb HDDS-7734. Implement symmetric SecretKeys lifescycle
management in SCM (#4194)
add 7dc1dee2a8 HDDS-8099. Make unlimited length the default in ozone debug
ldb. (#4404)
add c7de8d5871 HDDS-7741. [Snapshot] Delete keys in snapshot scope from
deleteTable during createSnapshot to accommodate snapshot garbage collection
(#4280)
add 07bb734e76 HDDS-8041. Let Ozone Client fail faster with wrong OM
address in URI. (#4325)
add 29acb2b188 HDDS-7548. [Snapshot] SnapDiff pagination and partial diff
report implementation (#4360)
add b25410d691 HDDS-8069. [Snapshot] Compaction DAG reconstruction logic
potentially crashing OM on startup (#4401)
add dc03783161 HDDS-8145. ReadReplicas should close client (#4387)
add f093f71314 HDDS-8158. Replication Manager: Make all handlers send
commands immediately instead of returning commands (#4399)
add ec62964e32 HDDS-8167. Inject MoveManager into ContainerBalancer (#4411)
add 0f1ba3f8a3 HDDS-7958. Ozone client not closed in integration tests
(#4373)
add 6c18d95477 HDDS-8122. [Snapshot] Disable RocksDB column family auto
compaction in SstFilteringService (#4400)
add 864ec707b7 HDDS-8183. Fix edge case where delimiter is empty string in
BucketEndpoint#get (#4410)
add 3068a31f53 HDDS-8160. [Snapshot] Skip the compaction entry from log
and DAG if compaction input and output files are exactly same. (#4416)
add 1c30a3528a HDDS-8149. Refactor the way to notify keyStoreFactory about
certificate renewed (#4390)
add c14200c84b HDDS-8111. ReplicationManager: Add
RatisMisReplicationHandler into rm.processUnderReplicatedContainer (#4414)
add 1236414c1c HDDS-8061. [Snapshot] Add SnapDiff client option to force a
full diff (#4398)
add 9619759632 HDDS-8171. Replicate commands could be sent to dead
maintenance nodes if the same index is being decommissioned (#4412)
add 8202b4184e HDDS-8204. Add testuser principals for all Ozone containers
(#4422)
add b570ca7c33 HDDS-8187. ReplicationManager: Datanode commands should be
sent to nodeManager directly (#4413)
add a76fb0ee58 HDDS-8205. Reorder OM nodes in HA acceptance tests (#4423)
add 9512c32c99 HDDS-8217. Fix OmBucketInfo#equals for comparing
defaultReplicationConfig (#4426)
add 22823a0012 HDDS-8221. Fix misplaced OzoneKey javadoc comment (#4430)
add c144a7d457 HDDS-8054. Fix NPE in metrics for failed volume (#4340)
add da8da7435c HDDS-8216. EC: OzoneClientConfig is overwritten in
ECKeyOutputStream (#4425)
add ceaaaae1a2 HDDS-8139. Datanodes should not drop block delete
transactions based on transaction ID (#4384)
add 130b7b6587 HDDS-8068. Fix Exception: JMXJsonServlet, getting attribute
RatisRoles of Hadoop:service=OzoneManager. (#4352)
add d0888381b7 HDDS-8222. EndpointBase#getBucket should handle
BUCKET_NOT_FOUND (#4431)
add 65268cc2c1 HDDS-7918. EC: ECBlockReconstructedStripeInputStream should
check for spare replicas before failing an index (#4441)
add 4a43f343ad HDDS-8168. Make deadlines inside MoveManager for move
commands configurable (#4415)
add 184bfc2118 HDDS-7883. [Snapshot] Accommodate FSO, key renames and
implement OMSnapshotPurgeRequest for SnapshotDeletingService (#4407)
add 030cf40fb1 HDDS-8172. ECUnderReplicationHandler should consider
commands already sent when processing the container (#4435)
add dc6df43e34 HDDS-8152. Reduce S3 acceptance test setup time (#4393)
add 598de2f9e0 HDDS-7975. Rebalance acceptance tests (#4437)
add 21e8ef0fca HDDS-8173. Fix to remove enrties from RocksDB after
container gets deleted. (#4445)
add 8e0c462c11 HDDS-8220. [Ozone-Streaming] Trigger volume check on
IOException in StreamDataChannelBase (#4428)
add 6dd80eba2c HDDS-8233. ReplicationManager: Throttle delete container
commands from over-replication handlers (#4447)
add 900f2db64a HDDS-8245. Info log for keyDeletingService when nonzero
number of keys are deleted. (#4451)
add b801c6860e HDDS-8154. Perf: Reuse Mac instances in S3 token validation
(#4433)
add 15117d8020 HDDS-8189. [Snapshot] renamedKeyTable should only track
keys in buckets that has at least one active snapshot. (#4436)
add 209a2e785d HDDS-8042. Display certificate issuer in cert list command.
(#4429)
add 3f5a80783a HDDS-7853. Add support for RemoveSCM in SCMRatisServer.
(#4358)
add 5a90593f20 HDDS-8148. Improve log for Pipeline creation failure (#4385)
add d9a48cc95d HDDS-8155. SCM Block deleting service add transaction
blocks for stale/dead/decommissioning DNs (#4403)
add cd37da0d08 HDDS-7874. Disable flaky unit test:
TestHddsSecureDatanodeInit.testCertificateRotationRecoverableFailure
add e38e4721a2 HDDS-8241. Transfer leader command doesn't work in secure
cluster (#4454)
add 823f56eaff HDDS-8098. Make Schema V3 as default in ozone debug ldb for
container db. (#4434)
add 181558a21a HDDS-8230. Let ReplicationManager decide the timeout for
commands in Datanodes (#4453)
add be6c20eb9b HDDS-8255. Remove unnecessary sleep at secure container
startup (#4457)
add 89e6ee1ef3 HDDS-8251. RocksIterator is not closed properly in snapshot
(#4461)
add 5756fd876f HDDS-7559. [Snapshot] Skip compaction tracking when no
Ozone snapshot exists (#4418)
add 68eb26d70e HDDS-8265. [Snapshot] Close ManagedColumnFamilyOptions
properly in OmSnapshotManager (#4465)
add 1eda443a1e HDDS-8176. [Snapshot] Rename variables to fix shadow
declaration warning with gcc (#4409)
add 23e0ce72d2 HDDS-8223. SCM delete block service shoud run wait for
safemode to exit. (#4432)
add a05543b1b0 HDDS-7122. Add validation for EC chunk size (#4463)
add b29efbe771 HDDS-8263. [JDK17] Bump guice to 5.1.0, maven-shade-plugin
to 3.4.1, remove guice-multibindings (#4462)
add 55076744d7 HDDS-8279. [Snapshot] Switch RocksDB Tools JNI to release
build (#4472)
add 1f29c4b5c3 HDDS-8244. Selective checks: handle change in junit.sh
(#4450)
add 191bb5f111 HDDS-8219. Run HA secure tests from s3g container (#4444)
add 11355a46f0 HDDS-8240. Disable JaCoCo for PRs and in forks (#4449)
add 386d1be18b HDDS-8128. Deduplicate the ops in RDBBatchOperation. (#4424)
add f863d9046b HDDS-8281. Add IntelliJ code style scheme import
recommendation in CONTRIBUTING.md (#4474)
add 50ca90e05e HDDS-7540. Document separate scheduled CI (#4392)
add 7357f752e4 HDDS-6064. Print proper JSON from `ozone debug ldb scan`
(#4420)
add 0d7a042917 HDDS-7697. Restrict change of bucket properties to owner
and admins in NativeACL (#4439)
add 5db68674de HDDS-5043. Consistently use user's short name in secure
mode (#2466)
add 01b7dc639b HDDS-7573. Use keyManager and trustManager provided by
keyStoreFactory in Ratis group (#4464)
add 31cc0bdc8a HDDS-8169. Delay Starting ContainerBalancer after SCM
failover (#4458)
add e8fbdaaca2 HDDS-8285. Eliminate leftover Guava Optional from
CacheValue (#4482)
add 33a0e61081 HDDS-8258. RocksIterator not closed properly in
KeyValueHandler.logBlocksIfNonZero (#4471)
add 915b11bea6 HDDS-8275. ManagedWriteBatch is not closed properly in
SCMHADBTransactionBuffer (#4476)
add 652459131a HDDS-8287. Add toString() method to ECContainerReplicaCount
(#4481)
add 1508370494 HDDS-8288. Upgrade moment.js to 2.29.4 (#4483)
add 1ed9a73af0 HDDS-8272. DBStore not closed properly in
ReconStorageContainerManagerFacade (#4469)
add 881a471c4e HDDS-8274. Intermittent timeout in acceptance-MR test setup
(#4475)
add e79a221556 HDDS-8257. RocksIterator not closed properly in
LegacyBucketHandler (#4468)
add 199da36ad6 HDDS-7842. java.lang.NullPointerException:
hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService.
(#4395)
add c74f19e8aa HDDS-8193. Ozone file systems to support Hadoop's
PathCapabilities interface (#4452)
add d9a1662494 HDDS-7781. Change Get Key Info to return HSync info (#4460)
add 428fe1de25 HDDS-7952. [Snapshot] Server side changes to add job status
and wait time to mimic the async behaviour for snapshot diff API. (#4438)
add 2a826133d6 HDDS-8195. RDBStore.getUpdatesSince() throws
RocksDBException: Requested array size exceeds VM limit (#4459)
add 012ecd3316 HDDS-8178. CertificateClient and KeyStoresFactory support
multiple Sub-CA certificates in the trust chain (#4442)
add 650e306c83 HDDS-6633. Refactor upgrade acceptance tests to reduce code
duplication. (#4456)
add 5ae810a7d8 HDDS-7855. Add hsync metrics in OM. (#4479)
add 05c8dbb674 HDDS-7994. Expose information via the ClusterState endpoint
on keys and Directories marked for deletion (#4406)
add cce3c7ea79 HDDS-8293. In Network Topology documentation remove DNS
configuration from Topology table mapping file. (#4485)
add b5137a0c78 HDDS-8290. [Snapshot] Merge entries from
snapshotRenamedKeyTable to next snapshot. (#4488)
add 4cb0e68efe HDDS-8276. EC file checksum calculation fails with
topology-aware read enabled (#4495)
add e8e34c2a91 HDDS-7914. [Snapshot] Block FS API access to deleted
(non-active) snapshots (#4419)
add 0a029def5a HDDS-8282. [Snapshot] Intermittent DB crash in
RocksDBCheckpointDiffer (#4492)
add 06433c0da9 HDDS-6961. [Snapshot] Bootstrapping slow followers/new
followers. (#3980)
add 48bfc26998 HDDS-8311. Fix list of supported units in Quota
documentation
add 17e5349bc2 HDDS-8134. Certificate clients are not correctly closed.
(#4380)
add 91227327b1 HDDS-8309. ReplicationManager: Basic Throttling of EC
Reconstruction commands (#4496)
add 278c4cdce0 HDDS-8330. [Doc] Fix misplaced doc for GDPR.zh.md (#4502)
add 54e4ca3b39 HDDS-8313. Support rocksdb WAL ttl and size limit
configuration (#4494)
add 57d66137fa HDDS-8323. [Snapshot] cancelAllBackgroundWork and close
listeners in RDBStore#close (#4498)
add 612cfd631a HDDS-8320. Fix ranger jackson version conflict (#4504)
add f9a240da19 HDDS-5463. [FSO] Recon Container API does not work
correctly with FSO. (#4182)
add 54a5a3c59c HDDS-8239. Backwards compatibility fix for new
RATIS_DATASTREAM port (#4500)
add e9592d2654 HDDS-8273. EC: Validate replication config at server-side
(#4507)
add 83969c0092 HDDS-8353. Mark
TestBlockDeletion#testContainerStatisticsAfterDelete as flaky
add 7b570dfa7a HDDS-8185. Implement JSON output for datanode list (#4508)
add 3ecd87cd7a HDDS-8334. ReplicationManager: Add nodes to exclude list if
they are overloaded (#4510)
add 3799bb228b HDDS-7817. Add HTTP and HTTPS ports to DatanodeDetails
(#4369)
add d10bbf53c1 HDDS-8089. Disable flaky test TestOmSnapshot#checkKey
add d91f619f22 HDDS-6335. Disable flaky test TestParentAcl#testKeyAcl
add a83ae0f1bd HDDS-5319. Disable flaky test
TestSCMUpdateServiceGrpcServer#testClientUpdateWithRevoke
add b509d37c08 HDDS-8355. Mark TestOMRatisSnapshots#testInstallSnapshot as
flaky
add 0c84ff1cdb HDDS-8291. Upgrade to Hadoop 3.3.5 (#4484)
add 4a8844151a HDDS-8324. DN data cache gets removed randomly asking for
data from disk (#4499)
add ac42bf0c77 HDDS-8354. Avoid WARNING about ObjectEndpoint#get (#4515)
add b7e08c1ed0 HDDS-8364. ReadReplicas may give wrong results with
topology-aware read enabled (#4522)
add eab67e71ef HDDS-8127. Exclude deleted containers from Recon container
count (#4440)
add e22a8f6103 HDDS-8135. Incorrect synchronization during certificate
renewal in DefaultCertificateClient. (#4381)
add 910eef097d HDDS-8325. Consolidate and refine RocksDB metrics of
services (#4506)
add 1a82bbd556 HDDS-8361. Useless object in TestOzoneBlockTokenIdentifier
(#4517)
add c90c4cb1c5 HDDS-8359. ReplicationManager: Fix
getContainerReplicationHealth() so that it builds ContainerCheckRequest
correctly (#4528)
add 34de64fdae HDDS-8358. Fix the space usage comparator in
ContainerBalancerSelectionCriteria (#4527)
add 4211d49eae HDDS-8368. ReplicationManager: Create ContainerReplicaOp
with correct target Datanode (#4532)
add 3adae01b52 HDDS-7974. [Snapshot] KeyDeletingService to be aware of
Ozone snapshots (#4486)
add 40e6875a7e HDDS-8304. [Snapshot] Reduce flakiness in
testSkipTrackingWithZeroSnapshot (#4487)
add d4d90c7d1a HDDS-8039. Allow container inspector to run from ozone
debug. (#4337)
add cb90fda213 HDDS-8035. Intermittent timeout in
TestOzoneManagerHAWithData.testOMHAMetrics (#4362)
add 05d9d85b8e HDDS-8237. [Snapshot] loadDb() used by SstFiltering service
creates extraneous directories. (#4446)
add 97f200f061 HDDS-8016. updated the ozone doc for linked bucket and
deletion async limitation (#4526)
add d9d61719bd HDDS-8374. Disable flaky unit test: TestContainerStateCounts
add 53d8fe0e57 update readme (#4535)
add 9b49f264b9 HDDS-8394. Upgrade Spring Framework to 5.3.26 (#4550)
add e5133e8f29 HDDS-8397. openKey need not setKeyArgs twice (#4552)
add b54cc253a0 HDDS-8170. Let ContainerBalancer consider EC containers for
balancing (#4542)
add 47d41da801 HDDS-8015. [Snapshot] Remove
RequestValidations#createValidationRequest (#4513)
add 6326fd5cb8 HDDS-8351. ReplicationManager: Use RM exclude list when
getting target nodes for reconstruction (#4520)
add bf35dc4cff HDDS-8337. ReplicationManager: MisReplicationHandler should
throw an exception if partially successful (#4536)
add 6207d7c536 HDDS-8367. SetQuota should have a check on usedNamespace
[NameSpacequota] (#4529)
add 45e8c5c45a HDDS-8383. Misreplication cannot be resolved with single
rack (#4539)
add 40692dbe5d HDDS-7862. [Snapshot] Wrap up TestOMSnapshotDeleteResponse
(#4548)
add 3873d889b6 HDDS-8286. Support SCM sub-ca certificate signed by leader
SCM's sub-ca certificate (#4493)
add 6da7aa086a HDDS-8228. Add metric for out of order/retried delete
transactions (#4533)
add eaf7f9a0c1 HDDS-8300. Enable topology aware replication and reads by
default. (#4489)
add 96d9582898 HDDS-8352. OM crash with NPE in OMKeyCommitRequest due to
missing user info (#4537)
add 03fdeaafaf HDDS-8302. Add a flag to disable hsync by default (#4512)
add 7075a967c4 HDDS-8346. ContainerBalancer should only move CLOSED
replicas (#4560)
add bbcb8101bd HDDS-8374. Fix flaky TestContainerStateCounts in Recon
(#4534)
add fc3f2861ef HDDS-7769. Implement client initiated lease recovery (#4234)
add f625a9f72c HDDS-8336. ReplicationManager: RatisUnderReplicationHandler
should partially recover the container if not enough nodes (#4561)
add fc97e4893c HDDS-8406. Add OM DB update data size and sequence metrics
(#4558)
add 974ebba7f7 HDDS-7758. Update Recon.zh.md doc for
ozone.recon.task.safemode.wait.threshold configuration (#4562)
add 8515c13a1e HDDS-8345. [Snapshot] Remove snapshot from
SnapshotChainManager in case of failure (#4525)
add 67f46e031e HDDS-8357. Exclude spring-jcl to avoid dependency conflict
(#4544)
add dd003040a4 HDDS-8317. [Snapshot] DirectoryDeletingService should clean
up Snapshot's deletedDirTable. (#4543)
add d34322f662 HDDS-8292. Inconsistent key name handling for FSO bucket
files. (#4531)
add c9424217cd HDDS-8242. Rename operation not working with FSO bucket
destination (#4478)
add 0991eab68b HDDS-8329. Performance Impact during container close when
still Client is writing (#4505)
add 617ef338cd HDDS-8370. Intermittent failure in
TestOzoneFileSystem#testListStatusWithIntermediateDir (#4564)
add c5912ff40d HDDS-8407. Intermittent failure in
TestOzoneFileSystem#testTrash (#4566)
add 2560e64515 HDDS-8363. Make SCM Ratis roles available in /prom endpoint
(#4521)
add e1ec24343b HDDS-8433. Mark TestOzoneManagerSnapshotAcl as flaky
add cffa3864b9 HDDS-7908. Support OM Metadata operation Generator in
`Ozone freon` (#4251)
add 600bbd5ced HDDS-8147. Introduce latency metrics for S3 Gateway
operations (#4383)
add b74faeff91 HDDS-8333. ReplicationManager: Allow partial EC
reconstruction if insufficient nodes available (#4579)
add 7506842d83 HDDS-8400. Expose rocksdb last sequence number through
metrics (#4557)
add dd489c7166 HDDS-8253. Set ozone.metadata.dirs to temporary dir if not
defined in S3 Gateway (#4455)
add 93cdbfebab HDDS-8429. Checkpoint is not closed properly in
OMDBCheckpointServlet (#4575)
add 0ee6f4a086 HDDS-8109. Seperate Ratis and EC MisReplication Handling
(#4577)
add d58d529d8d HDDS-8270. Measure checkAccess latency for Ozone objects
(#4467)
add 8f745fcc96 HDDS-8137. [Snapshot] SnapDiff to use tombstone entries in
SST files (#4376)
add 35c64efa3d HDDS-8453. Mark
TestDirectoryDeletingServiceWithFSO#testDirDeletedTableCleanUpForSnapshot as
flaky
add c2c393a8ac HDDS-8424: Preserve legacy bucket getKeyInfo behavior
(#4576)
add 97b18c5d27 HDDS-8385. Ozone can't process snapshot when service UID >
2097151 (#4580)
add 1cb935b064 HDDS-8458. Mark TestBlockDeletion#testBlockDeletion as flaky
add e61f348362 HDDS-7309. Enable by default GRPC between S3G and OM (#3820)
add f62476139c HDDS-8440. Ozone Manager crashed with ClassCastException
when deleting FSO bucket. (#4582)
add cb201556be HDDS-8446. Selective checks: handle change in ci.yaml
(#4587)
add 90fd5f2adc HDDS-8444. Increase timeout of CI build (#4586)
add 9d066ea265 HDDS-8355. Intermittent failure in
TestOMRatisSnapshots#testInstallSnapshot (#4592)
add 52db80f920 HDDS-8335. ReplicationManager: EC Mis and Under replication
handlers should handle overloaded exceptions (#4593)
add bee5454d7b HDDS-8445. Move PlacementPolicy back to SCM (#4588)
add 475963125b HDDS-7951. [Snapshot] Clean SnapDiff job and report table
(#4490)
add 0231bd7d05 HDDS-8261. Fail integration tests if RocksDB objects are
leaked (#4590)
add e7cd92c467 HDDS-8455. Om supports read only administrators. (#4565)
add cf3d0e7f8a HDDS-8017. Add check in ClosingContainerHandler for RM to
handle Missing Containers (#4598)
add de4c4c7f45 HDDS-6113. Mark
TestBlockOutputStreamWithFailuresFlushDelay#testWatchForCommitDatanodeFailure
as flaky
add 531ed34b55 HDDS-8461. Bump jetty to 9.4.51.v20230217 (#4594)
add 99bab79cef HDDS-8416. ReplicationManager: RatisUnderReplicationHandler
should filter sources by highest BCSID (#4599)
add efaa2b1656 HDDS-8474. XceiverClient not closed properly in
TestContainerStateMachineFailures (#4605)
add 7573c0934c HDDS-8421. Use FileChannel.size() in FilePerBlockStrategy
to reduce file operations (#4570)
add cf13e76520 HDDS-8478. Fix module name in ScmRoles IntelliJ run config
(#4610)
add b6033460c5 HDDS-8480. DB not closed properly in ContainerMapper (#4612)
add 0038dd6610 HDDS-8479. XceiverClient not closed properly in tests
(#4611)
add 5b1319a8c2 HDDS-8277. [Snapshot] Parsing Bug in SSTDumptool (#4470)
add 8d15d5bea7 HDDS-7789. [Snapshot] Provide API for snapDiff from OFS.
(#4491)
add e74fccb8f8 HDDS-7907. [hsync] KeyOutputStream is not thread safe.
(#4601)
add 70357d9c3f HDDS-8468. Fix native compilation for JDK 11 and 17 (#4603)
add cc1d2b3984 HDDS-7995. Support incremental snapshot on OM (#4294)
add c422d99d81 HDDS-8452. Scm server side RPC support for Scm
Decommissioning. (#4595)
add de3877fe7d HDDS-8466. Disable LegacyReplicationManager by default
(#4604)
add a44722f934 HDDS-7884. ofs: Recursive delete of path of symlink with
empty source fails (#4405)
add 677b6465c3 HDDS-8459. Ratis under replication handling in a rack aware
environment doesn't work (#4616)
add e50e0a6321 HDDS-8475. Unexpected IOException should be returned as
INTERNAL_ERROR. (#4609)
add 73809175c6 HDDS-8482. Let Ozone shell support multiple OM services
(#4613)
add 0221502ff1 HDDS-8433. Intermittent failure in
TestOzoneManagerSnapshotAcl (#4619)
add 298e7eee13 HDDS-7586. Allow user to create bucket with
non-s3-naming-convention (#4524)
add 6a7c0706cb HDDS-7828. Make Ozone fs rm symbolic links command support
posix behaviour (#4246)
add c616e8d63f HDDS-8417. Cap on queue of commands at datanode (#4618)
add 7df18f2647 HDDS-7551. Test in ec/ozonefs.robot is not executed in CI
(#4625)
add 14ab7783b9 HDDS-7412. Cannot delete empty OBS bucket if prefix of
other, non-empty bucket (#4606)
add c20881a9ef HDDS-8314. [Snapshot] SnapDiff job and compaction DAG/SST
file pruning synchronization (#4553)
add 16ee8c8b83 HDDS-8495. Fix permissions and path handling for ozone sh
token get command (#4620)
add 94d8219eea HDDS-8269. [Snapshot] Enable CI native building and testing
(#4596)
add c7797432b7 HDDS-8224. Ozone network topology does not create pipelines
based on rack aware policy. (#4623)
add 53b2a252e8 HDDS-7226. Allow existing nodes to be specified for Rack
Aware Policy. (#4614)
add fa831957aa HDDS-8458. Intermittent timeout in
TestBlockDeletion#testBlockDeletion (#4624)
add 5657009a3d Data Rendering code on SCM UI (#4608)
add 6eebe4523c HDDS-8369. Decommissioning with rack aware placement policy
does not replicate to correct rack. (#4556)
add ad2b6b6e52 HDDS-8471. Ensure replication processors use a single queue
for each iteration (#4627)
add 3858cd102b HDDS-8322. [Snapshot] Created various config for Snapdiff
and moved few configs from OzoneConfigKeys to OMConfigKeys (#4511)
add 5861766353 HDDS-8510. Intermittent timeout in
TestRootedOzoneFileSystem#testSnapshotDiff (#4632)
add ad9358fdc4 HDDS-8506. LeaseRecovery failing with NullPointer
exception. (#4631)
add df4df20371 HDDS-7965. Quota needs to be updated correctly for Hsync
(#4629)
add 90f3bba78e HDDS-8512. Bump Spring Framework to 5.3.27 (#4635)
add 4578a06353 HDDS-8435. [Snapshot] Handle Directory renames for FSO
Buckets. (#4607)
add be65ec1259 HDDS-8451. [Snapshot] Catch Exception and log it in
generateSnapshotDiffReport (#4600)
add b9057a2dcb HDDS-8505. ReplicationManager: Add configurable global
replication limit (#4636)
add 977a271b87 HDDS-8365. Admin SCM command to decommission SCM. (#4563)
add bc9258c0ea HDDS-7943. Link Datanode WebUIs from the Node Status table
on SCM WebUI (#4546)
add 643982e1ee HDDS-8515. Update README to include GitHub Discussions.
(#4641)
add 73d1ff104c HDDS-8243. [hsync] Add a CLI to recover lease (#4514)
add ad2d125d26 HDDS-8347. Investigate possible race conditions on
ContainerInfo in ContainerBalancer (#4643)
add ea4b01b93b HDDS-7935. [Snapshot] LRU Cache entries may get
evicted/closed during long running processes (#4568)
add ebff2f1705 HDDS-8496. S3 to return not found for object head/set when
keyinfo indicates a directory. (#4647)
add c708259b56 HDDS-8215. Recon - SCM DB Insights. (#4569)
add 74adc9ac29 HDDS-8494. Adjust replication queue limits for
out-of-service nodes (#4645)
add 46f0dc5c1d HDDS-8525. Provide more info in assertions (#4653)
add c78c07fda3 HDDS-8462. Key in deleteTable uniqueness (#4602)
add c4d348c30d HDDS-8393. [Recon] Include Roboto font using fontsource
instead (#4551)
add 1fda945f75 HDDS-8340. [Recon] Show the number of deleted containers in
UI (#4555)
add 8ae8e220a5 HDDS-8092. Mark
TestRecursiveAclWithFSO#testKeyDeleteAndRenameWithoutPermission as flaky (#4652)
add ecaa76b669 HDDS-8527. Update Ofs (HCFS) doc to describe shared tmp dir
(#4648)
add bf74f7857b HDDS-7546. [Snapshot] Throw exception if the SnapshotDiff
size is huge (#4633)
add 5e5d04c9be HDDS-8531. Remove outdated/stale code around
TrashRecoverRequest. (#4656)
add 4dfe040a49 HDDS-8058. Gracefully handle invalid S3 request and
authorization headers (#4473)
add fe10a99bbb HDDS-8473. ReplicationManager: Clear
ContainerReplicaPendingOps when RM goes to running state (#4644)
add cdd96b8baf HDDS-7632. Add ability in Recon to track various Container
Sizes across Ozone cluster. (#4107)
add 69e3cf3505 HDDS-8541. CodecRegistry is not thread safe. (#4658)
add a455597815 HDDS-8548. Migrate tests in hdds-erasurecode to JUnit5
(#4662)
add 831f608d28 HDDS-8166. [Snapshot] Add a config to enable or disable
Ozone snapshot feature on OM (#4642)
add 759433c2ec HDDS-8499. Add mechanism to notify threads when OM double
buffer flushed. (#4621)
add 2feb7388aa HDDS-8554. Thread pool size needs to be decreased in
different order in ReplicationSupervisor (#4669)
add 9fa304f502 HDDS-8550. Mark testOverWriteKeyWithAndWithOutVersioning as
flaky
add 7f7ec2397a HDDS-8557. Disable testOverWriteKeyWithAndWithOutVersioning
add 75f5a28f37 HDDS-8542. In RDBTable, add a put method using RocksDB
ByteBuffer APIs. (#4666)
add 2281e06e92 HDDS-8521. Fix SCM certificate count check during OM &
Datanode startup. (#4654)
add cf6592d421 HDDS-7889. Mark TestOzoneSnapshotRestore as flaky
add 5d68ac93f9 HDDS-8484. Allow Ozone volume name to have special
characters (#4630)
add 9f1b0fc51e HDDS-8493. Intermittent timeout in
TestDecommissionAndMaintenance#testSCMHandlesRestartForMaintenanceNode (#4671)
add 72b5de7548 HDDS-8518. Admin SCM decommission command primordial node
support (#4649)
add 2dfac8a0f2 HDDS-8533. Ozone Cluster components (SCM, OM, DNs) fails to
start with JDK 17 (#4659)
add ec62011ab3 HDDS-8514. Improve ozone admin cert list subcommand count
limits (#4646)
add de922e26b6 HDDS-8551. Fix the generic type of CodecRegistry.getCodec
(#4667)
add 297c3c1eb3 HDDS-7814. Vault-based remote S3 secret store (#4389)
add 557449e916 HDDS-8546. Ozone debug chunkinfo command throws
GroupMismatchException for closed container (#4661)
add 5765ba8851 HDDS-8390. [Snapshot] Added active snapshots check before
returning diff report to client and in diff report generation to fail fast.
(#4617)
add 740be38cfa HDDS-8390. [Snapshot] Added active snapshots check before
returning diff report to client and in diff report generation to fail fast.
(#4617)
add 13a4f81353 HDDS-8289: Speed up FSO ListKeys, skip skipToFirst (#4650)
add df43d549dd HDDS-8539. Container DB open, but not found in
DatanodeStoreCache (#4672)
add 0fcfe212e1 HDDS-8179. Datanode decommissioning blocked due to
non-empty replica of deleting container (#4685)
add 290e5f2ba1 HDDS-8550. Intermittent failure in
testOverWriteKeyWithAndWithOutVersioning (#4686)
add 5e8b806f05 HDDS-8382. Upgrade Ratis to 2.5.1 (#4664)
add 81f1364e31 HDDS-8532. Add config for factor of scaling up replication
queue/threads in decommissioning nodes (#4687)
add 71087dd092 HDDS-8064. [Snapshot] Clean up FSO deletedDirTable as well
during snapshot creation (#4651)
new 43aaa9545a HDDS-7734. Implement symmetric SecretKeys lifescycle
management in SCM (#4194)
new 42e2cb302e HDDS-7830. SCM API for OM and Datanode to get secret keys
(#4345)
new f966961491 HDDS-7831. Use symmetric secret key to sign and verify
token (#4417)
new b1880fffae HDDS-8003. E2E integration test cases for block tokens
(#4547)
new ba51dbacdf HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (783c90f72e)
\
N -- N -- N refs/heads/HDDS-7733-Symmetric-Tokens (ba51dbacdf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ci.md | 12 +-
.github/workflows/ci.yml | 16 +-
CONTRIBUTING.md | 9 +
README.md | 6 +-
dev-support/ci/selective_ci_checks.bats | 24 +
dev-support/ci/selective_ci_checks.sh | 4 +
.../hadoop/hdds/scm/client/HddsClientUtils.java | 27 +-
.../ozone/client/io/BadDataLocationException.java | 9 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 42 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 9 +-
.../TestECBlockReconstructedStripeInputStream.java | 76 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 13 -
.../java/org/apache/hadoop/hdds/StringUtils.java | 26 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 15 +-
.../hdds/client/ReplicationConfigValidator.java | 52 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 37 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 4 +-
.../apache/hadoop/hdds/scm/RemoveSCMRequest.java | 112 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 8 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 7 +
.../protocol/StorageContainerLocationProtocol.java | 6 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 23 -
.../hdds/upgrade/BelongsToHDDSLayoutVersion.java | 28 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 6 +-
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 20 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 64 +-
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 162 +++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 38 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 32 +
.../apache/hadoop/ozone/lease/LeaseManager.java | 31 +
.../org/apache/hadoop/util/ClosableIterator.java | 27 +-
.../java/org/apache/hadoop/util/MetricUtil.java | 12 +
.../common/src/main/resources/ozone-default.xml | 216 +++-
.../hadoop/hdds/client/TestReplicationConfig.java | 10 +-
.../client/TestReplicationConfigValidator.java | 114 +-
.../org/apache/hadoop/hdds/utils/TestIOUtils.java | 18 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../helpers/BlockDeletingServiceMetrics.java | 15 +-
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/helpers/DatanodeIdYaml.java | 39 +-
.../ozone/container/common/impl/ContainerSet.java | 7 +-
.../common/statemachine/DatanodeConfiguration.java | 25 +-
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../common/statemachine/StateContext.java | 10 +
.../commandhandler/DeleteBlocksCommandHandler.java | 30 +-
.../DeleteContainerCommandHandler.java | 32 +-
.../ReconstructECContainersCommandHandler.java | 1 -
.../SetNodeOperationalStateCommandHandler.java | 14 +-
.../common/states/datanode/InitDatanodeState.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 15 +-
.../container/common/utils/DatanodeStoreCache.java | 16 +
.../ozone/container/common/volume/DbVolume.java | 6 +-
.../ozone/container/common/volume/HddsVolume.java | 5 +-
.../container/common/volume/MutableVolumeSet.java | 41 +-
.../container/common/volume/StorageVolume.java | 48 +-
.../container/common/volume/VolumeInfoMetrics.java | 9 +-
.../KeyValueContainerMetadataInspector.java | 157 ++-
.../ozone/container/keyvalue/KeyValueHandler.java | 17 +-
.../container/keyvalue/TarContainerPacker.java | 4 +-
.../container/keyvalue/helpers/ChunkUtils.java | 51 +
.../keyvalue/impl/FilePerBlockStrategy.java | 12 +-
.../keyvalue/impl/StreamDataChannelBase.java | 33 +-
.../background/BlockDeletingService.java | 2 +-
.../StaleRecoveringContainerScrubbingService.java | 8 +-
.../ozone/container/metadata/DatanodeTable.java | 5 +
.../metadata/SchemaOneDeletedBlocksTable.java | 5 +
.../container/ozoneimpl/ContainerController.java | 4 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/replication/ReplicationServer.java | 40 +
.../replication/ReplicationSupervisor.java | 173 ++-
.../protocol/commands/CloseContainerCommand.java | 30 +
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 11 +-
.../container/common/TestDatanodeStateMachine.java | 2 +-
.../common/helpers/TestContainerUtils.java | 17 +-
.../common/helpers/TestDatanodeIdYaml.java | 99 +-
.../common/impl/TestContainerPersistence.java | 43 +
.../common/statemachine/TestStateContext.java | 3 +-
.../TestDeleteContainerCommandHandler.java | 47 +-
.../container/common/volume/TestHddsVolume.java | 19 +
.../common/volume/TestReservedVolumeSpace.java | 25 +-
.../container/common/volume/TestVolumeSet.java | 3 +-
.../keyvalue/ContainerTestVersionInfo.java | 5 +
.../container/keyvalue/TestKeyValueContainer.java | 7 +
.../TestKeyValueContainerIntegrityChecks.java | 3 +-
.../TestKeyValueContainerMetadataInspector.java | 197 +++-
.../container/keyvalue/TestTarContainerPacker.java | 5 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 13 +
.../ReplicationSupervisorScheduling.java | 10 +-
.../replication/TestReplicationConfig.java | 14 +
.../replication/TestReplicationSupervisor.java | 135 ++-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 12 +-
hadoop-hdds/docs/content/concept/Recon.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 1 +
hadoop-hdds/docs/content/feature/Quota.md | 13 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 13 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 16 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 14 +-
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/Ofs.md | 38 +-
hadoop-hdds/docs/content/security/GDPR.md | 1 -
hadoop-hdds/docs/content/security/GDPR.zh.md | 3 -
.../docs/content/security/SecuringDatanodes.zh.md | 5 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 2 +-
.../docs/content/security/SecuringTDE.zh.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
.../ozone/erasurecode/TestCodecRegistry.java | 8 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 6 +-
.../rawcoder/TestCodecRawCoderMapping.java | 20 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 12 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 10 +-
.../rawcoder/TestNativeXORRawCoder.java | 10 +-
.../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 4 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 2 +-
.../erasurecode/rawcoder/TestRawCoderBase.java | 22 +-
.../rawcoder/TestRawErasureCoderBenchmark.java | 2 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 2 +-
.../framework/dev-support/findbugsExcludeFile.xml | 4 -
...inerLocationProtocolClientSideTranslatorPB.java | 17 +
.../hadoop/hdds/security/OzoneSecretManager.java | 3 +-
.../hdds/security/ssl/MonitoringTimerTask.java | 77 --
.../security/ssl/PemFileBasedKeyStoresFactory.java | 84 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 29 +-
.../security/ssl/ReloadingX509TrustManager.java | 18 +-
.../symmetric/DefaultSecretKeySignerClient.java | 16 +-
.../x509/certificate/client/CertificateClient.java | 6 +
.../client/CertificateNotification.java | 6 +-
.../client/DefaultCertificateClient.java | 74 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 1 -
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 21 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 59 +-
.../apache/hadoop/hdds/utils/FaultInjector.java | 38 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 53 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 32 +-
.../hdds/utils/PrometheusMetricsSinkUtil.java | 2 +-
.../hadoop/hdds/utils/RDBSnapshotProvider.java | 233 ++++
...sDBStoreMBean.java => RocksDBStoreMetrics.java} | 143 +--
.../hadoop/hdds/utils/db/ByteArrayCodec.java | 2 +-
.../hadoop/hdds/utils/db}/ByteStringCodec.java | 39 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 129 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 58 +-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 27 +-
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 19 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 227 +++-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 33 +-
.../hadoop/hdds/utils/db/RDBCheckpointUtils.java | 70 ++
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 19 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 159 +--
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 12 +-
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 42 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 40 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 23 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 9 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 26 +-
.../src/main/resources/webapps/static/ozone.css | 31 +-
.../ssl/TestPemFileBasedKeyStoresFactory.java | 5 -
.../security/ssl/TestReloadingX509KeyManager.java | 46 +-
.../ssl/TestReloadingX509TrustManager.java | 46 +-
.../token/TestOzoneBlockTokenSecretManager.java | 6 -
.../certificate/authority/TestDefaultCAServer.java | 264 ++---
.../client/CertificateClientTestImpl.java | 15 +-
.../client/TestDefaultCertificateClient.java | 117 +-
.../client/TestDnCertificateClientInit.java | 4 +-
.../hadoop/hdds/utils/TestRDBSnapshotProvider.java | 239 ++++
.../org/apache/hadoop/hdds/utils/db/TestCodec.java | 130 +++
.../hadoop/hdds/utils/db/TestCodecRegistry.java | 69 ++
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 66 ++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 140 ++-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 111 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 22 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 61 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../interface-client/src/main/proto/hdds.proto | 12 +
.../hdds/utils/db/managed/ManagedRocksDB.java | 7 -
.../db/managed/ManagedRocksObjectMetrics.java | 9 +-
hadoop-hdds/pom.xml | 2 +-
hadoop-hdds/rocks-native/pom.xml | 27 +-
hadoop-hdds/rocks-native/src/CMakeLists.txt | 4 +-
.../utils/db/managed/ManagedSSTDumpIterator.java | 186 ++-
.../hdds/utils/db/managed/ManagedSSTDumpTool.java | 17 +-
.../src/main/patches/rocks-native.patch | 83 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 9 +
.../ozone/rocksdb/util/ManagedSstFileReader.java | 208 +++-
.../org/apache/ozone/rocksdb/util/RdbUtil.java | 11 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 589 ++++++----
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 217 ++--
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 11 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 0
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 58 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 14 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 58 +-
.../container/AbstractContainerReportHandler.java | 21 +-
.../scm/container/CloseContainerEventHandler.java | 57 +-
.../scm/container/balancer/ContainerBalancer.java | 33 +-
.../balancer/ContainerBalancerConfiguration.java | 26 +-
.../ContainerBalancerSelectionCriteria.java | 64 +-
.../container/balancer/ContainerBalancerTask.java | 34 +-
.../hdds/scm/container/balancer/MoveManager.java | 88 +-
.../algorithms/SCMContainerPlacementRackAware.java | 255 +++-
.../algorithms/SCMContainerPlacementRandom.java | 3 +-
....java => CommandTargetOverloadedException.java} | 4 +-
.../replication/ContainerHealthResult.java | 7 +-
.../replication/ContainerReplicaPendingOps.java | 51 +-
.../replication/ECContainerReplicaCount.java | 112 +-
.../replication/ECMisReplicationHandler.java | 60 +-
.../replication/ECOverReplicationHandler.java | 75 +-
.../replication/ECUnderReplicationHandler.java | 343 ++++--
.../replication/LegacyReplicationManager.java | 1 +
.../replication/MisReplicationHandler.java | 110 +-
.../replication/OverReplicatedProcessor.java | 26 +-
.../replication/RatisContainerReplicaCount.java | 2 +-
.../replication/RatisMisReplicationHandler.java | 34 +-
.../replication/RatisOverReplicationHandler.java | 94 +-
.../replication/RatisUnderReplicationHandler.java | 151 ++-
.../container/replication/ReplicationManager.java | 599 +++++++---
.../replication/ReplicationManagerUtil.java | 93 ++
.../container/replication/ReplicationQueue.java | 34 +-
.../replication/UnderReplicatedProcessor.java | 24 +-
.../replication/UnhealthyReplicationHandler.java | 11 +-
.../replication/UnhealthyReplicationProcessor.java | 58 +-
.../health/ClosingContainerHandler.java | 7 +
.../health/ECReplicationCheckHandler.java | 7 +-
.../replication/health/EmptyContainerHandler.java | 13 +-
.../health/RatisReplicationCheckHandler.java | 50 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 69 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 8 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 4 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 +
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 12 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 13 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 31 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 19 +-
.../apache/hadoop/hdds/scm/ha/SCMHAMetrics.java | 116 ++
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 43 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 3 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 11 -
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 105 --
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 8 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 19 +
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 3 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 66 +-
.../pipeline/InsufficientDatanodesException.java | 7 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 29 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 14 +-
...inerLocationProtocolServerSideTranslatorPB.java | 15 +
.../hdds/scm/server/SCMClientProtocolServer.java | 28 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 18 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 35 +-
.../hdds/scm/server/StorageContainerManager.java | 159 ++-
.../main/resources/webapps/scm/scm-overview.html | 10 +-
.../src/main/resources/webapps/scm/scm.js | 51 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 23 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 17 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 8 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 9 +-
.../container/TestCloseContainerEventHandler.java | 43 +-
.../scm/container/TestContainerReportHandler.java | 104 ++
.../TestIncrementalContainerReportHandler.java | 122 ++
.../container/balancer/TestContainerBalancer.java | 88 +-
.../balancer/TestContainerBalancerTask.java | 223 +++-
.../scm/container/balancer/TestMoveManager.java | 33 +-
.../algorithms/TestContainerPlacementFactory.java | 9 +-
.../TestSCMContainerPlacementRackAware.java | 209 ++++
.../container/replication/ReplicationTestUtil.java | 179 ++-
.../TestContainerReplicaPendingOps.java | 23 +
.../replication/TestECContainerReplicaCount.java | 16 +
.../replication/TestECMisReplicationHandler.java | 61 +-
.../replication/TestECOverReplicationHandler.java | 142 ++-
.../replication/TestECUnderReplicationHandler.java | 373 +++++-
.../replication/TestLegacyReplicationManager.java | 17 +-
.../replication/TestMisReplicationHandler.java | 124 +-
.../replication/TestOverReplicatedProcessor.java | 58 +-
.../TestRatisContainerReplicaCount.java | 23 +
.../TestRatisMisReplicationHandler.java | 27 +-
.../TestRatisOverReplicationHandler.java | 121 +-
.../TestRatisUnderReplicationHandler.java | 106 +-
.../replication/TestReplicationManager.java | 563 +++++++--
.../replication/TestUnderReplicatedProcessor.java | 121 +-
.../health/TestClosingContainerHandler.java | 49 +
.../health/TestRatisReplicationCheckHandler.java | 107 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 6 +
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 263 +++++
.../hadoop/hdds/scm/ha/TestSCMHAMetrics.java | 69 ++
.../apache/hadoop/hdds/scm/ha}/package-info.java | 7 +-
.../scm/metadata/TestSCMMetadataStoreImpl.java | 67 --
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 8 +
.../scm/pipeline/TestPipelinePlacementFactory.java | 41 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 43 +-
.../scm/server/TestSCMClientProtocolServer.java | 95 ++
.../server/TestSCMUpdateServiceGrpcServer.java | 1 +
.../testutils/ReplicationNodeManagerMock.java | 9 +-
.../org/apache/ozone/test/GenericTestUtils.java | 15 +
.../hdds/scm/cli/ContainerOperationClient.java | 9 +
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 19 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 16 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 8 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 9 +
.../hadoop/ozone/client/OzoneClientFactory.java | 1 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 11 +-
.../client/checksum/ECFileChecksumHelper.java | 3 +-
.../checksum/ReplicatedFileChecksumHelper.java | 2 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 15 +-
.../ozone/client/protocol/ClientProtocol.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneClient.java | 5 +
.../hadoop/ozone/client/TestOzoneECClient.java | 41 +-
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 38 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 82 +-
.../apache/hadoop/ozone/om/S3InMemoryCache.java | 82 ++
.../org/apache/hadoop/ozone/om/S3SecretCache.java | 13 +-
.../apache/hadoop/ozone/om/S3SecretManager.java | 10 +-
.../hadoop/ozone/om/exceptions/OMException.java | 2 +-
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 42 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 25 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 36 +-
.../hadoop/ozone/om/helpers/OmKeyRenameInfo.java | 91 --
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 14 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 16 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 34 +-
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 253 ----
.../ozone/snapshot/SnapshotDiffReportOzone.java | 194 ++++
.../ozone/snapshot/SnapshotDiffResponse.java | 6 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 6 +
.../dev-support/checks/_mvn_unit_report.sh | 18 +-
hadoop-ozone/dev-support/checks/build.sh | 19 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 6 +
hadoop-ozone/dev-support/checks/junit.sh | 11 +-
hadoop-ozone/dev-support/checks/native_check.sh | 50 +
hadoop-ozone/dev-support/intellij/ozone-site.xml | 11 +-
.../intellij/runConfigurations/ScmRoles.xml | 2 +-
hadoop-ozone/dist/pom.xml | 8 +-
.../dist/src/main/compose/ozone-ha/docker-config | 2 +
.../dist/src/main/compose/ozone-ha/test.sh | 15 +-
.../src/main/compose/ozone-om-ha/docker-config | 3 +
.../dist/src/main/compose/ozone/docker-config | 5 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 3 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 53 +
.../src/main/compose/ozonesecure-ha/docker-config | 38 +-
.../load.sh => ozonesecure-ha/docker-config-scm4} | 13 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 52 +-
.../src/main/compose/ozonesecure-mr/docker-config | 4 +-
.../non-ha/load.sh => ozonesecure-vault/.env} | 19 +-
.../compose/ozonesecure-vault/docker-compose.yaml | 138 +++
.../docker-config | 12 +-
.../krb5.conf | 0
.../{ozonesecure-ha => ozonesecure-vault}/test.sh | 22 +-
.../src/main/compose/ozonesecure/docker-config | 10 +-
.../dist/src/main/compose/ozonesecure/test.sh | 8 -
hadoop-ozone/dist/src/main/compose/test-all.sh | 6 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 50 +
.../dist/src/main/compose/upgrade/README.md | 93 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 76 +-
.../main/compose/upgrade/compose/ha/docker-config | 19 +-
.../src/main/compose/upgrade/compose/ha/load.sh | 4 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 16 +-
.../compose/upgrade/compose/non-ha/docker-config | 4 +
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +
.../compose/upgrade/compose/{ha => om-ha}/.env | 0
.../compose/{ha => om-ha}/docker-compose.yaml | 16 +-
.../upgrade/compose/{ha => om-ha}/docker-config | 4 +
.../compose/upgrade/compose/{ha => om-ha}/load.sh | 4 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 12 +-
.../dist/src/main/compose/upgrade/testlib.sh | 139 +--
.../upgrade/upgrades/manual-upgrade/driver.sh | 4 +-
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 --
.../callbacks/1.3.0/callback.sh} | 19 +-
.../{1.1.0-1.2.0 => callbacks/common}/callback.sh | 48 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 34 +-
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 580 -> 580 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 278 -> 576 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 146 -> 146 bytes
hadoop-ozone/dist/src/main/keytabs/httpfs.keytab | Bin 458 -> 616 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 142 -> 142 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 278 -> 576 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 296 -> 606 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 434 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 586 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 606 -> 908 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 154 -> 920 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 19 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 3 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 10 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
.../src/main/smoketest/admincli/datanode.robot | 6 +
.../admincli/{scmha.robot => scmrole.robot} | 2 +-
.../dist/src/main/smoketest/basic/links.robot | 11 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 8 +-
.../dist/src/main/smoketest/commonlib.robot | 15 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../src/main/smoketest/debug/ozone-debug.robot | 2 +-
.../dist/src/main/smoketest/ec/basic.robot | 44 +
.../dist/src/main/smoketest/ec/ozonefs.robot | 61 -
.../main/smoketest/omha/om-leader-transfer.robot | 73 ++
.../dist/src/main/smoketest/omha/testOMHA.robot | 61 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 6 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 38 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 14 +
.../src/main/smoketest/scmha/primordial-scm.robot | 60 +
.../main/smoketest/scmha/scm-decommission.robot | 60 +
.../main/smoketest/scmha/scm-leader-transfer.robot | 48 +
.../src/main/smoketest/security/admin-cert.robot | 4 +-
.../main/smoketest/security/ozone-secure-fs.robot | 22 +-
.../smoketest/security/ozone-secure-tenant.robot | 2 +-
.../smoketest/security/ozone-secure-token.robot | 18 +-
.../main/smoketest/snapshot/snapshot-acls.robot | 2 +-
.../src/main/smoketest/snapshot/snapshot-sh.robot | 4 +-
.../smoketest/upgrade/check-finalization.robot | 50 +
.../src/main/smoketest/upgrade/check-mlv.robot | 41 -
.../dist/src/main/smoketest/upgrade/finalize.robot | 6 +-
.../dist/src/main/smoketest/upgrade/prepare.robot | 1 +
hadoop-ozone/dist/src/shell/ozone/ozone | 26 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 5 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 8 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 157 ++-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 97 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 155 +++
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 10 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 10 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 22 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 159 +--
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 64 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 105 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 7 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 7 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 8 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 62 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 15 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 8 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 296 ++++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +
.../hadoop/fs/ozone/contract/OzoneContract.java | 13 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 4 +
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 9 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 9 +-
.../metrics/TestSCMContainerManagerMetrics.java | 11 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 36 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 31 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 17 +
.../org/apache/hadoop/ozone/RatisTestHelper.java | 9 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 2 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 25 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 298 ++---
.../hadoop/ozone/TestStorageContainerManager.java | 35 +-
.../ozone/TestStorageContainerManagerHelper.java | 40 +-
.../ozone/client/TestOzoneClientFactory.java | 8 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 2 +
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 +
.../client/rpc/TestBlockDataStreamOutput.java | 2 +
.../ozone/client/rpc/TestBlockOutputStream.java | 2 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 3 +-
.../rpc/TestCloseContainerHandlingByClient.java | 7 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 323 +++---
.../rpc/TestContainerReplicationEndToEnd.java | 15 +-
.../client/rpc/TestContainerStateMachine.java | 14 +-
.../TestContainerStateMachineFailureOnRead.java | 14 +-
.../rpc/TestContainerStateMachineFailures.java | 110 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +
.../rpc/TestContainerStateMachineStream.java | 2 +
.../client/rpc/TestDeleteWithSlowFollower.java | 31 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 4 +-
.../client/rpc/TestFailureHandlingByClient.java | 2 +
.../rpc/TestFailureHandlingByClientFlushDelay.java | 2 +
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 +
.../rpc/TestMultiBlockWritesWithDnFailures.java | 2 +
.../client/rpc/TestOzoneAtRestEncryption.java | 101 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 7 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 +
.../rpc/TestOzoneClientRetriesOnExceptions.java | 2 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 198 ++--
.../TestOzoneRpcClientWithKeyLatestVersion.java | 5 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 6 +-
.../ozone/client/rpc/TestWatchForCommit.java | 215 ++--
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/container/TestContainerReplication.java | 2 +
.../ozone/container/TestECContainerRecovery.java | 11 +-
.../commandhandler/TestBlockDeletion.java | 89 +-
.../TestCloseContainerByPipeline.java | 2 +
.../commandhandler/TestCloseContainerHandler.java | 19 +-
.../commandhandler/TestDeleteContainerHandler.java | 5 +-
.../TestRefreshVolumeUsageHandler.java | 69 +-
.../container/ozoneimpl/TestOzoneContainer.java | 7 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 29 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 3 +-
.../container/server/TestContainerServer.java | 6 +
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 146 +++
.../apache/hadoop/ozone/debug}/package-info.java | 17 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 4 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 7 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 8 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 74 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 7 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 7 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 14 +-
.../ozone/om/TestBucketLayoutWithOlderClient.java | 9 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 138 +--
.../ozone/om/TestContainerReportWithKeys.java | 5 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 11 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 7 +-
.../org/apache/hadoop/ozone/om/TestLDBCli.java | 517 +++++----
.../hadoop/ozone/om/TestListKeysWithFSO.java | 7 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 7 +-
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 8 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 370 +++++-
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 8 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 439 ++++++-
.../ozone/om/TestOMStartupWithBucketLayout.java | 17 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 8 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 8 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 39 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 15 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 23 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 11 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 117 +-
.../hadoop/ozone/om/TestOmSnapshotDisabled.java | 103 ++
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 107 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 32 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 10 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 6 +-
.../hadoop/ozone/om/TestOzoneManagerHAMetrics.java | 104 ++
.../ozone/om/TestOzoneManagerHAWithData.java | 93 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 36 +-
.../om/TestOzoneManagerListVolumesSecure.java | 544 +++++++++
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 10 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 293 ++---
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 9 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 7 +
.../om/multitenant/TestMultiTenantVolume.java | 37 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 14 +-
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 56 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 10 +-
.../om/snapshot/TestOzoneSnapshotRestore.java | 21 +-
.../ozone/parser/TestOzoneHARatisLogParser.java | 8 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 7 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 29 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 17 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +
.../ozone/scm/TestStorageContainerManagerHA.java | 213 ++--
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 2 +
.../hadoop/ozone/scm/TestXceiverClientManager.java | 311 ++---
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 175 +--
.../scm/node/TestDecommissionAndMaintenance.java | 18 +-
.../org/apache/hadoop/ozone/scm}/package-info.java | 7 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 8 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 7 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 181 +++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 77 +-
.../impl/StatemachineImplTestUtil.java | 17 +-
.../src/test/resources/log4j.properties | 1 +
.../src/test/resources/ozone-site.xml | 20 +
.../src/main/proto/OmClientProtocol.proto | 38 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 40 +-
.../ozone/om/codec/OmDBDiffReportEntryCodec.java | 28 +-
.../ozone/om/codec/OmKeyRenameInfoCodec.java | 55 -
hadoop-ozone/ozone-manager/pom.xml | 35 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 51 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 216 +++-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 43 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 5 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 6 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 332 ++++--
.../apache/hadoop/ozone/om/OmMetadataReader.java | 28 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 23 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 723 ++++++++++--
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 81 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 23 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 12 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 272 +++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 21 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 64 +-
.../hadoop/ozone/om/SstFilteringService.java | 9 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 24 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 74 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 32 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 42 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 34 +-
.../OmRatisSnapshotProvider.java} | 122 +-
.../{snapshot => ratis_snapshot}/package-info.java | 2 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 2 +-
.../ozone/om/request/OMClientRequestUtils.java | 52 +
.../om/request/bucket/OMBucketCreateRequest.java | 15 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 7 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 28 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 3 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../request/file/OMFileCreateRequestWithFSO.java | 2 +-
.../om/request/file/OMRecoverLeaseRequest.java | 239 ++++
.../om/request/key/OMAllocateBlockRequest.java | 3 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 23 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 69 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 46 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 36 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 38 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 69 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 11 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 3 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 3 +-
.../om/request/key/OMTrashRecoverRequest.java | 143 ---
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 5 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 7 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 6 +-
.../S3InitiateMultipartUploadRequest.java | 7 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../multipart/S3MultipartUploadAbortRequest.java | 5 +-
.../S3MultipartUploadCommitPartRequest.java | 6 +-
.../S3MultipartUploadCompleteRequest.java | 14 +-
.../om/request/s3/security/OMSetSecretRequest.java | 2 +-
.../om/request/s3/security/S3GetSecretRequest.java | 3 +-
.../request/s3/security/S3RevokeSecretRequest.java | 3 +-
.../tenant/OMSetRangerServiceVersionRequest.java | 3 +-
.../s3/tenant/OMTenantAssignAdminRequest.java | 3 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 8 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 34 +-
.../request/s3/tenant/OMTenantDeleteRequest.java | 5 +-
.../s3/tenant/OMTenantRevokeAdminRequest.java | 3 +-
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 15 +-
.../security/OMCancelDelegationTokenRequest.java | 3 +-
.../security/OMGetDelegationTokenRequest.java | 3 +-
.../security/OMRenewDelegationTokenRequest.java | 3 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 62 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 7 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 13 +-
.../OMSnapshotPurgeRequest.java} | 54 +-
.../request/upgrade/OMFinalizeUpgradeRequest.java | 4 +-
.../om/request/validation/RequestValidations.java | 26 +-
.../om/request/volume/OMVolumeCreateRequest.java | 3 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 6 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 5 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 9 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 29 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 3 +-
.../om/response/file/OMRecoverLeaseResponse.java | 80 ++
.../response/key/AbstractOMKeyDeleteResponse.java | 22 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 56 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 26 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 16 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 2 +
.../ozone/om/response/key/OMKeyPurgeResponse.java | 31 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 20 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 30 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 2 +
.../om/response/key/OMKeysRenameResponse.java | 17 +-
.../om/response/key/OMTrashRecoverResponse.java | 68 --
.../snapshot/OMSnapshotCreateResponse.java | 62 +-
.../OMSnapshotMoveDeletedKeysResponse.java | 44 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 145 +++
.../hadoop/ozone/om/s3/LocalS3StoreProvider.java | 21 +-
.../hadoop/ozone/om/s3/S3SecretCacheProvider.java | 57 +
.../om/s3/S3SecretStoreConfigurationKeys.java | 42 +-
.../hadoop/ozone/om/s3/S3SecretStoreProvider.java | 25 +-
.../apache/hadoop/ozone/om/s3/package-info.java} | 11 +-
.../om/service/AbstractKeyDeletingService.java | 457 ++++++++
.../ozone/om/service/DirectoryDeletingService.java | 298 +----
.../ozone/om/service/KeyDeletingService.java | 212 +---
.../ozone/om/service/OpenKeyCleanupService.java | 21 +-
.../ozone/om/service/SnapshotDeletingService.java | 369 +++++-
.../om/service/SnapshotDiffCleanupService.java | 254 ++++
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 144 +++
.../hadoop/ozone/om/snapshot/PersistentList.java | 6 +-
.../hadoop/ozone/om/snapshot/PersistentMap.java | 7 +
.../hadoop/ozone/om/snapshot/PersistentSet.java | 4 +-
.../om/snapshot/RequireSnapshotFeatureState.java | 24 +-
.../RequireSnapshotFeatureStateAspect.java | 122 ++
.../ozone/om/snapshot/RocksDbPersistentList.java | 49 +-
.../ozone/om/snapshot/RocksDbPersistentMap.java | 71 +-
.../ozone/om/snapshot/RocksDbPersistentSet.java | 15 +-
.../hadoop/ozone/om/snapshot/SnapshotDiffJob.java | 207 ++++
.../ozone/om/snapshot/SnapshotDiffManager.java | 1226 +++++++++++++++-----
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 137 +++
.../hadoop/ozone/om/snapshot/package-info.java | 2 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 18 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 68 +-
.../src/main/resources/META-INF/aop.xml | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 8 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 2 +-
.../hadoop/ozone/om/TestOMTenantCreateRequest.java | 182 +++
.../hadoop/ozone/om/TestOmMetadataManager.java | 5 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 131 ++-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 32 +-
.../hadoop/ozone/om/TestSstFilteringService.java | 4 +-
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 150 +++
...TestOzoneManagerDoubleBufferWithOMResponse.java | 8 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 4 +-
.../ozone/om/request/OMRequestTestUtils.java | 138 +--
.../request/TestOMClientRequestWithUserInfo.java | 14 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 4 +
.../request/bucket/TestOMBucketCreateRequest.java | 111 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 21 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 5 +-
.../om/request/file/TestOMRecoverLeaseRequest.java | 314 +++++
.../TestOMDirectoriesPurgeRequestAndResponse.java | 5 +-
.../om/request/key/TestOMKeyCommitRequest.java | 83 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 4 +
.../om/request/key/TestOMKeyCreateRequest.java | 10 +
.../key/TestOMKeyPurgeRequestAndResponse.java | 114 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 17 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 8 +
.../s3/security/TestS3GetSecretRequest.java | 10 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 179 +--
.../snapshot/TestOMSnapshotDeleteRequest.java | 19 +-
.../TestOMSnapshotPurgeRequestAndResponse.java | 312 +++++
.../request/volume/TestOMVolumeCreateRequest.java | 72 ++
.../volume/TestOMVolumeSetQuotaRequest.java | 27 +
.../ozone/om/response/TestCleanupTableInfo.java | 5 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeyCommitResponse.java | 24 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 12 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeyRenameResponse.java | 11 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 5 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 3 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 19 +-
.../s3/multipart/TestS3MultipartResponse.java | 5 +-
.../snapshot/OMSnapshotResponseTestUtil.java | 69 ++
.../snapshot/TestOMSnapshotCreateResponse.java | 171 ++-
.../snapshot/TestOMSnapshotDeleteResponse.java | 82 +-
.../ozone/om/service/TestKeyDeletingService.java | 71 ++
.../om/service/TestOpenKeyCleanupService.java | 24 +-
.../om/service/TestSnapshotDeletingService.java | 3 +
.../om/service/TestSnapshotDiffCleanupService.java | 389 +++++++
.../om/snapshot/SnapshotFeatureEnabledUtil.java | 35 +-
.../TestRequireSnapshotFeatureStateAspect.java | 59 +
.../om/snapshot/TestRocksDbPersistentList.java | 122 +-
.../om/snapshot/TestRocksDbPersistentMap.java | 129 +-
.../om/snapshot/TestRocksDbPersistentSet.java | 147 ++-
.../ozone/security/TestAWSV4AuthValidator.java | 29 +-
.../security/TestOmCertificateClientInit.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 1 +
.../security/acl/TestOzoneAdministrators.java | 80 ++
.../security/acl/TestOzoneNativeAuthorizer.java | 9 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 43 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 62 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 10 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 86 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 274 +++--
.../fs/ozone/CapableOzoneFSOutputStream.java | 9 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 23 +-
.../org/apache/hadoop/fs/ozone/OzoneFsDelete.java | 206 ++++
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 10 +
.../hadoop/fs/ozone/OzonePathCapabilities.java | 51 +
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 12 +
.../apache/hadoop/fs/ozone/TestOzoneFsShell.java | 68 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 16 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 16 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 16 +-
hadoop-ozone/pom.xml | 10 +-
hadoop-ozone/recon-codegen/pom.xml | 10 +-
.../recon/schema/UtilizationSchemaDefinition.java | 14 +
hadoop-ozone/recon/pom.xml | 9 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 17 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 5 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 8 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 30 +-
.../hadoop/ozone/recon/api/BlocksEndPoint.java | 174 +++
.../ozone/recon/api/ClusterStateEndpoint.java | 56 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 147 ++-
.../ozone/recon/api/UtilizationEndpoint.java | 55 +-
.../ozone/recon/api/handlers/BucketHandler.java | 6 +
.../ozone/recon/api/handlers/EntityHandler.java | 4 +-
.../recon/api/handlers/LegacyBucketHandler.java | 145 +--
.../recon/api/handlers/RootEntityHandler.java | 4 +-
.../recon/api/handlers/VolumeEntityHandler.java | 4 +-
.../recon/api/types/ClusterStateResponse.java | 52 +
.../api/types/ContainerBlocksInfoWrapper.java | 82 ++
.../recon/api/types/ContainerStateCounts.java | 62 +
.../recon/api/types/DeletedContainerInfo.java | 142 +++
.../api/types/FileSizeDistributionResponse.java | 8 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 6 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 18 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 9 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 9 +-
.../scm/ReconStorageContainerManagerFacade.java | 71 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 14 +-
.../impl/StorageContainerServiceProviderImpl.java | 10 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 38 +-
.../ozone/recon/tasks/ContainerSizeCountTask.java | 366 ++++++
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 4 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 19 +
.../main/resources/webapps/recon/WEB-INF/web.xml | 4 +
.../webapps/recon/ozone-recon-web/api/db.json | 1 +
.../webapps/recon/ozone-recon-web/package.json | 1 +
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 18 +-
.../recon/ozone-recon-web/public/index.html | 1 -
.../ozone-recon-web/public/static/media/LICENSE | 15 +
.../webapps/recon/ozone-recon-web/src/index.tsx | 3 +
.../src/views/overview/overview.tsx | 11 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 52 +
.../hadoop/ozone/recon/api/TestBlocksEndPoint.java | 281 +++++
.../ozone/recon/api/TestContainerEndpoint.java | 489 +++++++-
.../ozone/recon/api/TestContainerStateCounts.java | 165 +++
.../hadoop/ozone/recon/api/TestEndpoints.java | 93 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 2 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 2 +-
.../recon/api/TestTotalOpenContainerCount.java | 396 -------
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../TestUtilizationSchemaDefinition.java | 24 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 5 +
.../recon/tasks/TestContainerKeyMapperTask.java | 221 +++-
.../recon/tasks/TestContainerSizeCountTask.java | 157 +++
.../ozone/recon/tasks/TestNSSummaryTask.java | 15 +-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 25 +-
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 27 +-
.../{recon-codegen => s3-secret-store}/pom.xml | 53 +-
.../S3SecretRemoteStoreConfigurationKeys.java | 59 +
.../hadoop/ozone/s3/remote/package-info.java} | 10 +-
.../remote/vault/VaultS3SecretStorageProvider.java | 23 +-
.../ozone/s3/remote/vault/VaultS3SecretStore.java | 210 ++++
.../s3/remote/vault/VaultS3SecretStoreBuilder.java | 174 +++
.../ozone/s3/remote/vault/auth/AppRoleAuth.java | 57 +
.../hadoop/ozone/s3/remote/vault/auth/Auth.java | 25 +-
.../ozone/s3/remote/vault/auth/AuthType.java | 51 +
.../s3/remote/vault/auth/DirectTokenAuth.java | 25 +-
.../ozone/s3/remote/vault/auth/package-info.java} | 10 +-
.../ozone/s3/remote/vault/package-info.java} | 11 +-
.../s3/remote/vault/VaultS3SecretStoreTest.java | 193 +++
hadoop-ozone/s3gateway/pom.xml | 5 +
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 32 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +
.../ozone/s3/commontypes/EncodingTypeObject.java | 4 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 44 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 72 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 6 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 377 ++++--
.../s3/signature/AuthorizationV4HeaderParser.java | 41 +-
.../s3/signature/AuthorizationV4QueryParser.java | 135 ++-
.../ozone/s3/signature/StringToSignProducer.java | 97 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 31 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 29 +
.../hadoop/ozone/s3/endpoint/TestListParts.java | 8 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 20 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 3 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 13 +-
.../signature/TestAuthorizationV4HeaderParser.java | 24 +
.../signature/TestAuthorizationV4QueryParser.java | 210 +++-
.../s3/signature/TestStringToSignProducer.java | 194 +++-
.../ozone/admin/scm/DecommissionScmSubcommand.java | 61 +
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 211 ++--
.../org/apache/hadoop/ozone/debug/DBScanner.java | 447 +++----
.../apache/hadoop/ozone/debug/LeaseRecoverer.java | 84 ++
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 7 +
.../apache/hadoop/ozone/debug/ReadReplicas.java | 167 +--
.../ozone/debug/container/ContainerCommands.java | 5 +
.../ozone/debug/container/InspectSubcommand.java | 74 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 118 +-
.../ozone/freon/ClosedContainerReplicator.java | 24 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 452 ++++++++
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 25 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 18 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 8 +-
.../hadoop/ozone/shell/token/GetTokenHandler.java | 9 +
.../hadoop/ozone/shell/token/TokenOption.java | 9 +-
.../ozone/shell/volume/ListVolumeHandler.java | 2 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 2 +-
.../ozone/scm/TestDecommissionScmSubcommand.java | 108 ++
.../shell/TestOzoneAddressClientCreation.java | 12 +
pom.xml | 28 +-
905 files changed, 34893 insertions(+), 11284 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/RemoveSCMRequest.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentList.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java
(63%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentSet.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java
(55%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentSet.java
=>
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestIOUtils.java
(75%)
delete mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretCache.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/FaultInjector.java
(59%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/{RocksDBStoreMBean.java
=> RocksDBStoreMetrics.java} (70%)
rename
hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata =>
framework/src/main/java/org/apache/hadoop/hdds/utils/db}/ByteStringCodec.java
(63%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointUtils.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestCodec.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestCodecRegistry.java
rename hadoop-hdds/{common =>
server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java (91%)
rename hadoop-hdds/{common =>
server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java
(100%)
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{AllSourcesOverloadedException.java
=> CommandTargetOverloadedException.java} (88%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAMetrics.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreMetrics.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAMetrics.java
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot
=>
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha}/package-info.java
(87%)
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSCMMetadataStoreImpl.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3InMemoryCache.java
delete mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java
delete mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
create mode 100644 hadoop-ozone/dev-support/checks/native_check.sh
copy hadoop-ozone/dist/src/main/compose/{upgrade/compose/non-ha/load.sh =>
ozonesecure-ha/docker-config-scm4} (73%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{upgrade/compose/non-ha/load.sh =>
ozonesecure-vault/.env} (73%)
mode change 100755 => 100644
create mode 100644
hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-compose.yaml
copy hadoop-ozone/dist/src/main/compose/{ozonesecure =>
ozonesecure-vault}/docker-config (93%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-ha =>
ozonesecure-vault}/krb5.conf (100%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-ha =>
ozonesecure-vault}/test.sh (76%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/.env
(100%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha =>
om-ha}/docker-compose.yaml (94%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha =>
om-ha}/docker-config (93%)
copy hadoop-ozone/dist/src/main/compose/upgrade/compose/{ha => om-ha}/load.sh
(89%)
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
copy hadoop-ozone/dist/src/main/compose/upgrade/{compose/non-ha/load.sh =>
upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh} (61%)
rename
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/{1.1.0-1.2.0
=> callbacks/common}/callback.sh (52%)
rename hadoop-ozone/dist/src/main/smoketest/admincli/{scmha.robot =>
scmrole.robot} (98%)
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/ozonefs.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/omha/om-leader-transfer.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/scmha/primordial-scm.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/scmha/scm-decommission.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/check-mlv.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
copy
hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot
=>
integration-test/src/test/java/org/apache/hadoop/ozone/debug}/package-info.java
(74%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotDisabled.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetrics.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java
copy
hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot
=>
integration-test/src/test/java/org/apache/hadoop/ozone/scm}/package-info.java
(87%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java
=>
hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
(67%)
delete mode 100644
hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{snapshot/OzoneManagerSnapshotProvider.java
=> ratis_snapshot/OmRatisSnapshotProvider.java} (65%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{snapshot
=> ratis_snapshot}/package-info.java (94%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/{key/OMKeyPurgeRequest.java
=> snapshot/OMSnapshotPurgeRequest.java} (54%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMRecoverLeaseResponse.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/LocalS3StoreProvider.java
(58%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretCacheProvider.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretStoreConfigurationKeys.java
(50%)
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/S3SecretStoreProvider.java
(65%)
copy
hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
=>
ozone-manager/src/main/java/org/apache/hadoop/ozone/om/s3/package-info.java}
(82%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureState.java
(63%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureStateAspect.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMTenantCreateRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotResponseTestUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java
copy
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/SnapshotFeatureEnabledUtil.java
(56%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRequireSnapshotFeatureStateAspect.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzonePathCapabilities.java
create mode 100644
hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsShell.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/BlocksEndPoint.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerBlocksInfoWrapper.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DeletedContainerInfo.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerSizeCountTask.java
create mode 100644
hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/public/static/media/LICENSE
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestBlocksEndPoint.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java
delete mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerSizeCountTask.java
copy hadoop-ozone/{recon-codegen => s3-secret-store}/pom.xml (58%)
create mode 100644
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/S3SecretRemoteStoreConfigurationKeys.java
copy
hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
=>
s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/package-info.java}
(82%)
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java
=>
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStorageProvider.java
(65%)
create mode 100644
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStore.java
create mode 100644
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStoreBuilder.java
create mode 100644
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/AppRoleAuth.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AllSourcesOverloadedException.java
=>
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/Auth.java
(60%)
create mode 100644
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/AuthType.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java
=>
hadoop-ozone/s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/DirectTokenAuth.java
(57%)
copy
hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
=>
s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/auth/package-info.java}
(82%)
copy
hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
=>
s3-secret-store/src/main/java/org/apache/hadoop/ozone/s3/remote/vault/package-info.java}
(82%)
create mode 100644
hadoop-ozone/s3-secret-store/src/test/java/org/apache/hadoop/ozone/s3/remote/vault/VaultS3SecretStoreTest.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/DecommissionScmSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/LeaseRecoverer.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
copy hadoop-ozone/{dev-support/checks/build.sh =>
tools/src/main/resources/META-INF/services/org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory}
(74%)
mode change 100755 => 100644
create mode 100644
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestDecommissionScmSubcommand.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
