This is an automated email from the ASF dual-hosted git repository.
devmadhuu pushed a change to branch HDDS-11233
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 8787bc21038 HDDS-15881. Support setting storagePolicy via
SetBucketProperty (#11232)
add e2556a3aeac HDDS-16295. Bump cyclonedx-maven-plugin to 2.9.3 (#10970)
add b759fce51e7 HDDS-16165. Fix intermittent failure in
TestOzoneClientRetriesOnExceptions.testMaxRetriesByOzoneClient
add 1911e03d11e Enable OM S3 gRPC auth.
add d959514a979 HDDS-16277. Reuse CharsetEncoder/Decoder in StringCodec
(#11112)
add 1303c11a922 HDDS-16292. Bump actions/setup-java to 5.7.0 (#10975)
add 341b76d655a HDDS-14524. Add freon test that uses hfs API for both
writes and reads (#10651)
add 6f3abf61ac7 HDDS-16189. Return application/xml for S3 error responses
(#11031).
add 521e046314b HDDS-11839. Accept SCM node id in ozone admin scm transfer
(#11022)
add 9ff918bf2b3 HDDS-16305. Add version tag for pinned infra actions
(#11129)
add 36a80f5a0e6 HDDS-11421. `ozone dtutil get` to handle o3:// and ofs://
URLs (#11067)
add 77979b013ac HDDS-16098. Speed up TestStreamRead (#10961)
add e330a6aaa63 HDDS-16163. Fix TrashOzoneFileSystem error handling on
LEGACY buckets (#11016)
add 0df1bc25212 HDDS-15997. Fix QuotaRepairTask scan hang and partial
counts on worker failure (#10884)
add 1a8396cb560 HDDS-16100. Parameterize TestChunkInputStream (#11117)
add b23218a8cff HDDS-16251. Preserve signed empty Content-Type during
preprocessing (#11088).
add 6cc8f31d2fe HDDS-16074. Make Recon RECON_TASK_STATUS column upgrade
idempotent and fail on error. (#11060).
add 9fdadbb888d HDDS-16310. Avoid allocating a comparator per HeapEntry
comparison in ListIterator (#11137)
add acc685ec8d4 HDDS-16298. ECBlockChecksumComputer returns empty MD5 due
to duplicate digest() call (#11125)
add a439238a8cc HDDS-16322. ConfigurationSource.getClass loads property
key instead of configured class name (#11144)
add 1b6bb47767b HDDS-13310. Intermittent failure in
TestContainerOperations (#8770)
add a17ad8798a4 HDDS-16280. Avoid eager Preconditions message on hot paths
(#11131)
add 7e314c75661 HDDS-16331. Bump byte-buddy to 1.18.12 (#11155)
add f7a88e317e3 HDDS-16330. Bump zstd-jni to 1.5.7-15 (#11154)
add 17f5be015f5 HDDS-16033. Fix intermittent failures in
TestKeyLifecycleService (#11145)
add f56755819c0 HDDS-16328. Bump opentelemetry to 1.65.0 (#11152)
add 3e3eb1b0344 HDDS-16329. Bump avro to 1.12.2 (#11153)
add c87a5667526 HDDS-16332. Bump commons-collections4 to 4.6.0 (#11156)
add 4efa2943eca HDDS-16317. Rename TestKeyValueContainerIntegrityChecks
for naming convention (#11149)
add 2a33e5597e4 HDDS-16341. Bump awssdk to 2.54.2 (#11165)
add eec25d7870d HDDS-16344. Bump jackson to 2.21.6 (#11163)
add c898c6f7511 HDDS-16346. Bump jnr-posix to 3.2.2 (#11166)
add e564c45a1a4 HDDS-16343. Bump restrict-imports-enforcer-rule to 3.0.1
(#11162)
add f7a27ef1641 HDDS-16345. Bump commons-validator to 1.11.0 (#11164)
add 0047629eb7b HDDS-16348. StorageSize.parse fails on values with
surrounding whitespace (#11167)
add 1908e5e2928 HDDS-14439. Require Java 17 for server components (#9640)
add 5c70970a7c8 HDDS-16257. Avoid the flatten-copy in
OmMetadataManagerImpl.getExpiredOpenKeys' hsync branch (#11116)
add 74a7ab4b48f HDDS-16286. Use StaticMapping in MiniOzoneCluster topology
setup (#11158)
add 9bc3b590b9a HDDS-16256. Avoid the per-group list copy in
OMKeyRequest.filterOutBlocksStillInUse (#11115)
add 57072651c6a HDDS-15394. Sequential Reader of from and to snapshots for
full snapshot diff. (#11083)
add bf13abb3721 HDDS-16353. Pipeline violates the equals and hashCode
contract (#11179)
add fb30b6c1065 HDDS-16334. The maps in XceiverClientShortCircuit should
not use String as key. (#11178)
add e1506e441f0 HDDS-16373. Fix InterSCMGrpcClient checkpoint download
deadline unit (#11191)
add 0ec5ee2fa69 HDDS-16073. Implement ObjectParts listing and pagination
for completed multipart objects in GetObjectAttributes (#11020).
add 04547b2632b HDDS-16195. Reuse computed replicated sizes in
KeyDeletingService (#11028)
add e0043cae4c0 HDDS-16349. Use HashSet instead of TreeSet for the
per-datanode container index (#11168)
add 88ddb955d54 HDDS-16131. Intermittent failure in
TestBlockDeletingService#testBlockDeletionMetricsUpdatedProperlyAfterEachExecution
(#11013)
add c639efc7227 HDDS-16350. Avoid a point-get for the start-key check in
RDBTable.getRangeKVs (#11169)
add 0ef5184106a HDDS-16218. Fix SCM HA log messages that drop values and
stack traces (#11046)
add 54da65a9d45 HDDS-15356. Addendum: Make multi-buffer chunk checksum
allocation-free (#11008)
add 9d36126a46d HDDS-16311. Avoid the O(n^2) part-number lookup in S3
multipart complete (#11138)
add 0539afa0b28 HDDS-16214. Avoid per-group list copies in KeyManagerImpl
read paths (#11078)
add 16545021c4c HDDS-15141. Support `HMAC-SHA256` signature verification
for S3 chunked uploads (#11061).
add b8bc13d9a7e HDDS-16336. S3 GET inverted Range with start inside object
yields negative Content-Length (#11160).
add 0f0e73574b9 HDDS-16288. Replace MutableRate with lock-free
ConcurrentMutableRate on OM hot-path metrics (#11128)
add 27431b72117 HDDS-16385. Minor cleanups in NodeDecommissionManager
(#11211)
add e96df8e2003 HDDS-13887. [STS] Protobuf Plumbing for AssumeRole
Requests (#9254)
add 281a5b3630d HDDS-13848. [STS] Artifacts for Ranger to authorize STS
token (#9214)
add 7eaebe6e428 HDDS-13888. [STS] Introduce S3AssumeRoleRequest and
S3AssumeRoleResponse (#9276)
add b0ae8b59012 Merge branch 'master' into HDDS-13323-sts
add a8c868870a2 HDDS-13909. [STS] Introduce STSTokenIdentifier class
(#9277)
add b79a8dd4305 HDDS-13724. [STS] Part 1 - Create utility to convert IAM
policy to groupings of OzoneObj and Acls (#9239)
add 9d3c32fba57 HDDS-13950. [STS] Introduce STSTokenSecretManager to sign
STS tokens (#9323)
add b4ff104a446 HDDS-13925. [STS] Part 2 - Create utility to convert IAM
policy to groupings of OzoneObj and Acls (#9292)
add c63f444a461 HDDS-13961. [STS] Encrypt secretAccessKey in session token
(#9344)
add 934f4c18842 HDDS-13926. [STS] Part 3 - Create utility to convert IAM
policy to groupings of OzoneObj and Acls (#9306)
add 36852379d5c HDDS-13997. [STS] Plumbing for passing STS token through
S3 api processing (#9372)
add 8f7ec1011a4 HDDS-14066. [STS] Database updates for revoked STS tokens
(#9420)
add 3d177048952 HDDS-14067. [STS] Plumbing and CLI utility to revoke STS
token (#9435)
add 2062a06942e HDDS-13942. [STS] Part 4 - Create utility to convert IAM
policy to groupings of OzoneObj and Acls (#9315)
add ae4bdb30974 HDDS-14091. [STS] Deny access if STS token is found in
revoked table (#9445)
add 08186522ad5 HDDS-14011. [STS] Ranger interactions for STS tokens
(assumeRole and S3 api calls) (#9484)
add a47b0b47880 HDDS-14067. [STS] Plumbing and CLI utility to revoke STS
token (#9507)
add 0b2db9c80c9 HDDS-14373. [STS] Revoked STS token logic tweaks (#9604)
add c63f0096315 HDDS-14094. [STS] Background service to remove revoked
tokens that are past expiration from DB (#9468)
add 2cf4e918b8e HDDS-14364. [STS] Revoked permanent credential must render
all associated STS tokens useless (#9602)
add bf1453d2ec4 HDDS-13345. STS port and endpoint skeleton (#9343)
add 56da3888204 HDDS-14150. [STS] Connect STS Endpoint to Backend
Processing (#9673)
add bb2b540bd28 HDDS-14514. [STS] Revamp error handling in endpoint to
conform to AWS XML (#9674)
add 0f3b26b1b29 HDDS-14420. [STS] Add audit logging to endpoint and
OzoneManager for STS (#9687)
add 4b450b4980c HDDS-14472. [STS] Refactor constants and validation
methods to shared location (#9658)
add 7991b3abb6d HDDS-14538. [STS] Leader OM should generate access key and
secret and pass to others (#9697)
add 6ff971244c3 HDDS-14681. [STS] Support StringLike Condition operator in
IAM session policy and handle certain errors more gracefully (#9795)
add f9bec4bde8d HDDS-14711. [STS] Ensure accessKeyId is valid for
sessionToken (#9820)
add e63936bce0e HDDS-14716. [STS] Use MalformedPolicyDocument error code
for IAM Session Policy validation errors (#9823)
add f7c62ae2329 Merge remote-tracking branch 'origin/master' into
HDDS-13323-sts
add 5a4e49f1956 HDDS-14779. [STS] Part 1 - IAM Session Policy and
ListBucket improvements (#9894)
add 538a8f2a7b7 HDDS-14801. [STS] Part 2 - IAM Session Policy and
ListBucket improvements (#9895)
add 2a855ae8a72 HDDS-14808. [STS] Part 4 - IAM Session Policy and
ListBucket improvements (#9899)
add 1f723f7fbac HDDS-14851. [STS] Better Handling for PayloadTooLarge and
Small Perf Fix (#9938)
add 9b3577e8d79 HDDS-14847. [STS] Expose ExpiredToken Error (#9935)
add 9504881b5f8 HDDS-14809. [STS] Part 5 - IAM Session Policy and
ListBucket improvements (#9900)
add 1810c0d32e0 HDDS-14861. [STS] Fix Latent S3 API issue when ListBuckets
Missing a Required Permission (#9949)
add 37a224b2176 HDDS-14899. [STS] Updates to ACLs in
IamSessionPolicyResolver (#9977)
add 603a8abc375 Merge remote-tracking branch 'upstream/master' into
HDDS-13323-sts
add 56654627b20 HDDS-14935. [STS] Handle Latent Inconsistencies in S3 API
Acl Checks (#10009)
add fd0ae5c7f7e HDDS-15137. [STS] Ensure each S3 API has an associated S3
Action (#10197)
add 9f233336da2 HDDS-15182. Avoid extra read for modification time on
CopyObject/CopyPart (#10203)
add 11f957cea3a HDDS-15194. [STS] Update IamSessionPolicyResolver to
return S3 Actions (#10204)
add ef10673605c HDDS-15771. [STS] Add s3Action for expected bucket owner
condition checks (#10688)
add 79b82272595 HDDS-15861. [STS] Actions must not be sent in
RequestContext to Authorizer if feature flag is off (#10771)
add dc3b37d5701 HDDS-15984. [STS] Improve s3:prefix Condition handling and
reject unsupported AssumeRole parameters (#10875)
add 26380fdf76a Merge remote-tracking branch 'upstream/master' into
HDDS-13323-sts
add c41e9d6dcbd Merge remote-tracking branch 'upstream/master' into
HDDS-13323-sts
add fd455023fce HDDS-14810. [STS] Part 2 - STS Ranger Smoke Tests (#9902)
add a48cf12131f HDDS-14811. [STS] Part 3 - STS Ranger Smoke Tests (#9903)
add 113e3d0f100 HDDS-16187. [STS] Fix Latent S3 DeleteObjects Issue
(#11019)
add 3a56b7faa35 HDDS-15325. [STS] Polaris Smoke Test (#10315)
add 2b5240bf92f Merge remote-tracking branch 'upstream/master' into
HDDS-13323-sts
add ff6f7e0c5ec Merge remote-tracking branch 'upstream/master' into
HDDS-13323-sts
add d5a76c5204e HDDS-16240. [STS] Remove /sts from endpoint to enhance
compatibility (#11073)
add 5000f2835c5 HDDS-16186. Move sts robot tests out of misc suite (#11042)
add dc169f6e87c HDDS-16110. [STS] Update key in sts revocation table
(#11095)
add 5a0707f4baa HDDS-16472. [STS] Tighten session policy validation
(#11103)
add 30a353d780b HDDS-16371. [STS] Better messaging for invalid endpoint
path and prevent non-canonical token use (#11186)
add 0ab01a4777f HDDS-16313. [STS] Handle linked bucket and session
policies (#11196)
add 5bc5a1e362f HDDS-16290. [STS] Implement GetCallerIdentity API (#11121)
add 556c15e9117 Merge remote-tracking branch 'origin/HDDS-13323-sts' into
master
add 0c981849106 HDDS-16145. Fix race between DeletedBlockLogStateManager
transaction removal and SCM HA buffer flush (#11188)
add 1b9cbd1dd60 HDDS-16320. Recon incorrectly marks healthy EC containers
as REPLICA_MISMATCH by comparing checksums across different EC replica indexes.
(#11215)
add 37e75cb7787 HDDS-16092. Stop takeSnapshot from lowering the persisted
transaction index (#10953)
add 61cb7834055 HDDS-16229. Add RocksDB multiGetSkipCache for batched
table reads. (#11068)
add a394802ec93 HDDS-16314. Filter out blocks still in use before
enqueuing overwritten key versions in S3 MPU complete (#11135)
add bcf62c79a08 HDDS-15789. OM DB size should be reduced after compaction
(#11173)
add f9f61847685 HDDS-16135. Fix remaining intermittent failures in
TestKeyLifecycleService (#11202)
add 644bb55e6ba HDDS-16365. Pipeline.getReplicaIndexes() unnecessarily
create a new map. (#11194)
add 43ce3e0e473 HDDS-16409. Bump guava to 33.7.1-jre (#11208)
add 4cc03d119d7 HDDS-16411. Bump javassist to 3.33.0-GA (#11207)
add 75c1cca80e5 HDDS-16412. Bump sqlite-jdbc to 3.53.4.0 (#11205)
add 3f4e78e4c14 HDDS-16416. Bump actions/setup-java to 6.0.0 (#11210)
add dda118aec0b HDDS-15916. Native memory leak in ECKeyOutputStream
(#11180)
add 07fea7e1f5a HDDS-16359. Configure maven-resolver-transport-http retry
for CI (#11175)
add 5fb21276db1 HDDS-13218. Add integration tests for snapshot defrag
checkpoint footprint savings (#11094)
add d3843b157b1 HDDS-15857. Fix checksum calculation in KeyValueHandler
for variable-sized chunks (#10764)
add 6d15218437b HDDS-16372. Surface OM's detailed message for rejected
lifecycle configuration (#11189)
add b8a2ef3309a HDDS-16403. Replace deprecated
java.security.AccessControlException in HttpFSServer (#11231)
add c0629ebc6a0 HDDS-16392. Parse Proto2Codec values from CodecBuffer
instead of InputStream (#11233)
add 07e64df5c6f HDDS-15424. Support concurrent positional read (#11102)
add 80be70dcdc6 HDDS-16324. XceiverClientGrpc.streamRead should wait for
flow-control readiness via onReadyHandler instead of a 10 ms poll (#11195)
add 20e6a1c0f60 HDDS-15986. Report ozone local startup failures (#10934)
add 68f3d3c4ea9 HDDS-16406. Remove deprecated ThreadGroup.isDestroyed()
and ThreadGroup.destroy() (#11248)
add 23efbc7386a HDDS-16483. zizmor-action upgrade may break silently
(#11249)
add 2ce22d95300 HDDS-12869. Add some corner cases in create bucket (#11240)
add 7f33fc535ac HDDS-11717. Upgrade Spring Framework to 6.2.19, jOOQ to
3.19.37 (#11172)
add 4250410bc29 HDDS-16173. Add cluster utilization analysis for container
balancer CLI commands (#11037)
add 464470d6c27 HDDS-16484. Bump zizmorcore/zizmor-action to 0.6.4 (#11213)
add 9005b4af866 HDDS-16156. Create HTTP base dir under java.io.tmpdir
instead of the working directory (#10997)
add e6e62de12d5 HDDS-16351. FeatureProvider.Feature.of throws
NoSuchElementException instead of IllegalArgumentException (#11170)
add 9ddbaa4e50e HDDS-15925. Remove redundant InfoBucket RPC from OFS
getFileStatus by validating bucket layout server-side (#11226)
add 11ba2a1f13c HDDS-16360. Add liveness and readiness health endpoints to
S3 Gateway (#11224)
add fe4a01b7f9c HDDS-15698. Fix concurrent write and hsync race in
BlockOutputStreamEntryPool (#10830)
add bc7055ab43d HDDS-16363. Fix split-schema MPU size tracking in Recon
event handlers (#11181)
add 3cf259cae14 HDDS-16161. Prevent failed OM requests from changing
bucket usage (#11216)
add b5af13b8dc1 HDDS-16389. Add command to display snapshot counts per
bucket. (#11217)
add 0bc3461998d HDDS-11130. Intermittent failure in
TestSnapshotDeletingService#testSnapshotSplitAndMove (#11142)
add 60274badc16 HDDS-16405. ReplicationManagerReport should not be used
for transient state (#11230)
add 6dd41bede5c HDDS-16278. Reuse DecimalFormat in SCMNodeManager
statistics (#11113)
add 8f7fbbb26c5 HDDS-15225. Fix BlockDataStreamOutput.hsync() exception
handling (#11143)
add f1205dc8ca7 HDDS-16378. Skip creating completion marker file in
`upgrade-container-schema --dry-run` (#11198)
add 577f309591b HDDS-12688. Set test-friendly defaults in mini-cluster
(#10926)
add 896949610f9 HDDS-16425. Fail datanode startup cleanly on
initialization errors (#11246)
add 2c30d6f2924 HDDS-16138. Add compatibility coverage for SCM Ratis role
output over IPv6 (#11049)
add e8add5688da HDDS-16109. Update upgrade/xcompat tests to use 2.2.1
(#11133)
add 2e97c0fc07a HDDS-15485. Avoid ByteBuffer.wrap on
ChunkBuffer.put(byte[]) path (#11147)
add a21f3eeb8a9 HDDS-16374. Use the datanode-specific block deletion
interval during reconfiguration (#11193)
add a9c9caff033 HDDS-16084. Add ErrorProne to CI (#11056)
add 6cf6756734d HDDS-16511. Bump bouncycastle to 1.86 (#11267)
add 20ce16c82c9 HDDS-16507. Bump slf4j to 2.0.19 (#11268)
add 26557dfafb5 HDDS-16140. Qualify S3 Gateway path-style access over IPv6
(#11124)
add cc2fed27df5 HDDS-16508. Bump jline to 3.30.17 (#11269)
add aaa9901f6e7 HDDS-16509. Bump maven-install-plugin to 3.2.0 (#11265)
add 6aa2642cf23 HDDS-16510. Bump properties-maven-plugin to 1.3.1 (#11266)
add a5f33b9a1dc HDDS-16106. Add 2.1.2 to xcompat compatibility tests
(#11259)
add 170a4651ca5 HDDS-16515. Set payload hash in testIPv6HostInStringToSign
(#11273)
add cc125926f88 HDDS-16482. Fix flaky
TestBlockDataStreamOutput.closeAfterWriteFailureThrows (#11263)
add e4aab123a05 HDDS-16512. Bump actions/setup-java to 6.0.1 (#11270)
add e34086e3cac HDDS-16127. Document OmKeyInfo key location version
semantics (#11018)
add 8bc575ca156 HDDS-16514. Fix intermittent
TestReconContainerManager.testUpdateAndRemoveContainerReplica (#11274)
add f17df541a71 HDDS-16519. Bump maven-compiler-plugin to 3.16.0 (#11276)
add dd5236e9297 HDDS-16521. Bump byte-buddy to 1.18.13 (#11279)
add 39dfab3b6d8 HDDS-16522. Bump exec-maven-plugin to 3.6.4 (#11280)
add 0281ad7cd46 HDDS-16520. Bump build-helper-maven-plugin to 3.6.2
(#11278)
add 2bdfe5b28f7 HDDS-16524. Ignore error_prone_core 2.43 and later (#11283)
add 012c5e54ea3 HDDS-16525. Bump zstd-jni to 1.5.7-16 (#11284)
add 320d855c08b HDDS-16528. Bump maven-deploy-plugin to 3.2.0 (#11288)
add be473161a16 HDDS-16526. Bump netty to 4.1.138 (#11285)
add 3d330090129 HDDS-16527. Bump opentelemetry to 1.66.0 (#11286)
add c3f6cbf0149 HDDS-10788. Fix intermittent failure in
TestWatchForCommit.testWatchForCommitForRetryfailure (#11001)
add 1dff0050e07 HDDS-16199. Log skipped unhealthy containers at INFO
(#11041)
add c6f51fbec4c HDDS-16254. Deprecate ineffective
ozone.om.ratis.server.retry.cache.timeout (#11096)
add ecd7ff08e80 HDDS-16518. Intermittent timeout waiting on scm3 9860
(#11282)
add e9a0d941bfd HDDS-11646. Intermittent timeout in
TestXceiverClientMetrics (#11029)
add d18115d1ed4 HDDS-16523. Fix flaky openFileTable assertions in
TestHSync (#11281)
add 992c7cc3f27 HDDS-14072. Show structured test summary in CI result
(#10857)
add 906062ef325 HDDS-16196. Bump Apache Ratis to 3.3.1 (#11032)
add d7a2702212b HDDS-15828. Fix flaky TestDirectoryDeletingServiceWithFSO
snapshot cleanup (#11212)
add d0e0b10fc02 HDDS-16426. Fix thread-safety of positioned reads in
OzoneCryptoInputStream (#11251)
add 8621653ec90 HDDS-16504. Datanode startup can hang indefinitely when
Ratis group recovery stalls (#11264)
add aed7ceac7cc HDDS-16465. Validate snapshot chains before mutation
(#11253)
add 19ca844b89f HDDS-15918. Optimize TestSCMNodeManager by reducing
heartbeat configs (#10837)
add b5bf1d26fea HDDS-16315. Skip MD5 recomputation in CopyObject when
source ETag can be reused (#11136)
add 7fe5962ce04 HDDS-16220. Reduce OM bucket write-lock hold time during
FSO directory purge (#11118)
add 0fe7ffdff08 HDDS-16407. In OM serialize the Ratis reply eagerly so the
retry cache holds bytes, not the OMResponse tree (#11234)
add 718fe1ac733 HDDS-16421. Close the stream's block file when a request
is rejected on the stream (#11250)
add 8c6c3b61a2a HDDS-16550. Intermittent failure in
testSnapshotDiffWhenOmLeaderRestart (#11299)
add 6ae92dab30d HDDS-16506. [STS] Use Consistent DurationSeconds
Validation Message, minor doc updates (#11261)
add 5a73b2688ff HDDS-16342. Make dependabot update awssdk only monthly
(#11297)
add 8c6c9fa7229 HDDS-16139. Define and enforce OFS and O3FS IPv6 authority
contracts (#11108)
add f73fa1a506a HDDS-16400. Fix thread-safety of positioned reads in
OzoneFSInputStream (#11245)
add 968f12d343d HDDS-16561. Bump awssdk to 2.54.18 (#11300)
add 153902ae046 HDDS-16149. Remove Pair usage from
client/common/framework/container-service (#11132)
add 54ad584c7b7 HDDS-16529. Replace reflective OM follower routing with
typed dispatch (#11289)
add 23aeb7b6080 HDDS-15378. Add component labels for PRs based on changed
files (#10851)
add ee13c483d6d HDDS-11853. Avoid redundant UUID conversion in
XceiverClientRatis hot path (#11247)
add 74c0f155b2f HDDS-16420. Use partial cache for OmLifecycleConfiguration
and OmLifecycleScanState table (#11241)
add bc434323f30 HDDS-16393. Reject oversized Days values in
PutBucketLifecycle (#11290)
add b0c52667fe2 HDDS-12072. Add Robot test for repair om
update-transaction (#11303)
add 5f210a1365c HDDS-16560. [STS] Intermittent AccessDenied in GetObject
(#11306)
add 16d48c88861 HDDS-16562. Bump byte-buddy to 1.18.14 (#11301)
add eec0c14b39d HDDS-16589. Intermittent failure in
testOMProxyProviderFailoverOnConnectionFailure (#11312)
add 7b68fcd3414 HDDS-16282. Avoid O(n^2) key removal in S3 DeleteObjects
(#11126)
add 54c21025222 HDDS-16583. STS credential validation updates (#11305)
add d51f0e4e7a3 HDDS-16557. Enums with per-constant impl may not be
reproducible (#11311)
add 9539b07b3c8 HDDS-16375. Apply fileSize filter independently in Recon
fileCount endpoint (#11203)
add 4455796d0f1 HDDS-16582. Improve errorprone check (#11308)
add 8431e6133d6 HDDS-16357. Recon fileCount API does not normalize the
fileSize query parameter (#11174)
add 995e9389fdd HDDS-16498. Run sts and other Ranger tests in same split
(#11254)
add c0ec36033b6 HDDS-15931. Remove usage of internal netty metrics (#10846)
add 984edb2ed81 HDDS-16206. Start MiniOzoneCluster datanodes finalized
(#11111)
add d4d5ec65d5a HDDS-15913. Extract common Kerberos/MiniKDC setup from
serveral tests (#10873)
add 4ab81d2a2e2 HDDS-16126. Fence quota repair deltas by bucket objectID
(#11309)
add 95daf18cab3 HDDS-16423. Fix intermittent failure in
testContainerReconciliationFailureContainerScan
add aac57526e3e HDDS-16307. Construct listeners from separate host and
port values (#11130)
add e655dbdb9c4 HDDS-16590. Avoid storing RocksCheckpoint handle to avoid
use-after-free (#11317)
add 409c6820794 HDDS-16565. Update per-volume replication config keys for
consistency (#11304)
add b64c7fc0d76 HDDS-16387. ContainerBalancer status --verbose throws
IllegalStateException after a rejected start (#11275)
add 571a67a023d HDDS-16179. Estimate subcommand for container balancer
with profiles (#11255)
add 710832575fc HDDS-16601. Rename more test helpers and parents for
naming convention (#11325)
add 89c9a2529a9 HDDS-16593. Speed up TestKeyLifecycleService: remove 2 sec
wait (#11330)
add ad059968a55 HDDS-16604. S3 DeleteObjects returns 200 when OM rejects
the whole batch (#11327)
add 67044fa33a7 HDDS-16612. Bump zstd-jni to 1.5.7-17 (#11331)
add 54c1d08c999 HDDS-16617. Bump nimbus-jose-jwt to 10.10 (#11332)
add 33913b09971 HDDS-16618. Bump jnr-posix to 3.2.4 (#11333)
add fb2034c7870 HDDS-16607. Bump
apache/infrastructure-actions/allowlist-check to 1.0.4 (#11335)
add ba33ee9ac70 HDDS-16174. Add container balancer assessment command to
report cluster imbalance (#11256)
add 47d93f181cc HDDS-15209. Stabilize
TestRocksDBCheckpointDiffer#testDifferWithDB for RocksDB layout variance
(#11324)
add 36481b61e44 HDDS-16595. Bump download-maven-plugin to 2.1.0 (#11316)
add 119fc06fd8f HDDS-16613. Bump actions/labeler to 7.0.0 (#11336)
add f39a349e90d HDDS-16623. Datanode can be missing from NetworkTopology
after it recovers from DEAD (#11343)
add 4b020c16d65 HDDS-16597. Bump iceberg to 1.11.0 (#11318)
add 03d838e01e0 HDDS-16398. Fix intermittent AlreadyClosedException in
TestCommitWatcher (#11227)
add 8bdfef0b8fb HDDS-13365. OM support automatic table cache tracking and
cleanup (#10171)
add 7b7e453e9ab HDDS-16644. Bump jackson to 2.21.7 (#11352)
add ba05c2e3ad8 HDDS-16497. Support OM ThreadLocal context propagation for
Ratis read (#11257)
add 2468fa8b14d HDDS-16647. Set only server-side target version for
compile check (#11362)
add 907411afe71 HDDS-12950. Refactor TestBucketList to reduce duplicated
declarations (#11338)
add 3448f66a4df HDDS-16645. Replace usage of deprecated AccessController
in TestOmMixedWorkloadUnderDeletionBench (#11363)
add ce4953e56ed HDDS-16646. Remove listMaxSize from KeyLifecycleService.
(#11361)
add 12635659b13 HDDS-16605. Reduce noise from failed copy-jar in
integration check (#11326)
add 46a0ec802b9 HDDS-16551. Remove duplicate heatmap feature check in
Recon v2 UI (#11323)
add b15c871db90 HDDS-16642. Hide heatmap navbar tab until feature status
is confirmed in Recon v2 (#11360)
add db63057eaf2 HDDS-16635. Remove redundant InfoBucket RPC from S3
GetBucketLifecycleConfiguration (#11355)
add 8638daf7f3e HDDS-16636. Remove redundant InfoBucket RPC from S3
ListParts (#11358)
add 24e06a40c16 HDDS-15835. Count incomplete MPU parts in quota repair
(#10795)
add 2b5874b743b HDDS-16226. Tool to obtain snapshot trapped bytes for a
bucket. (#11122)
add 4317f21bc85 HDDS-16603. Remove redundant infoBucket RPC call in S3
tagging APIs (#11339)
add a0119aaebd2 HDDS-16637. Remove redundant InfoBucket RPC from S3
ListMultipartUploads (#11359)
add 0f20ca7b287 HDDS-16655. Skip shading in integration (filesystem) check
(#11371)
add 8c249d0a048 HDDS-15142. Support HMAC-SHA256 trailer signature
verification for S3 chunked uploads (#11222)
add 985d7a94e54 HDDS-16099. Speed up TestStreamBlockInputStream (#11337)
add ecafca6f6f6 HDDS-16630. Update the network topology when a datanode
re-registers with new ports or a new version (#11353)
add d5d4df4d4b6 HDDS-16395. Add lock-free StreamBlock pread (#11314)
add 4a832912fea HDDS-16591. Reject not implemented S3 subresource APIs
(#11313)
add f35c48c9f64 HDDS-8280. Upgrade Jetty 9.4 to 12.0.38 (EE8) (#11367)
add aade154f8dd HDDS-16648. Tune HTTP settings for CI (#11375)
add 29758e4ac1d HDDS-16441. Orphaned snapshot diff task marks a
resubmitted job DONE with another run's metadata (#11321)
add 037a367ceb1 HDDS-16650. Reuse bucket metadata in S3 lifecycle PUT and
DELETE (#11372)
add fd90ad1b7a5 HDDS-16530. Allow Recon service principal to call OM
getDBUpdates (#11382).
add f7bfdf19b63 HDDS-16659. Reuse destination OzoneBucket in streaming S3
CopyObject (#11383)
add 63f93f7ca79 HDDS-16323. StreamBlockInputStream: refill the pre-read
window in bulk instead of once per response (#11159)
add 60ce9947960 HDDS-16619. Intermittent failure in
TestReconAsPassiveScm.testDatanodeRegistrationAndReports (#11341)
add 177fe51472f HDDS-13303. Fix TestSnapshotDeletingServiceIntegrationTest
(#11310)
add 29b98532703 S3 header auth expiry in 15min and correct date validation.
add 03f73b4b07a HDDS-16705. Bump zstd-jni to 1.5.7-20 (#11390)
add ae281be3149 HDDS-16707. Bump commons-lang3 to 3.21.0 (#11394)
add 317c423af3e HDDS-16706. Bump slf4j to 2.0.20 (#11392)
add 225727d1890 HDDS-2821. Include datanode hostname/IP in logs in
addition to UUID (#11381)
add 898bb4310f2 HDDS-14337. Fix SCM startup failure near year boundary
when seeding localId (#11184)
add 7f7cb3f5597 HDDS-14018. Avoid repeated operations in
SCMNodeManager#getNodeStatistics (#11292)
add 69c51c4c2ca HDDS-16710. Bump axios to 1.20.0 (#11378)
add 2e51e156477 HDDS-16711. Bump moment to 2.31.0 (#11373)
add ac9a57ddc1b HDDS-16658. Reuse source key details when opening input
stream in S3 CopyObject (#11396)
add 4c99fd8fe12 HDDS-16654. Replace usage of deprecated finalize() in OM
(#11376)
add 77d582975a6 HDDS-16631. Fix chooseRandom for rack names with common
prefixes (#11401)
add 037f9161a33 HDDS-16704. ReadBlock fails with IllegalStateException
when a response is shorter than responseDataSize (#11402)
add 65c4078829d HDDS-15089. Support S3 per request read consistency
(#11252)
add ec64e924c40 HDDS-16300. Allow OM to dynamically reconfigure its SCM
node list without a restart (#11218)
add 4d1e5d7b49e HDDS-16713. Bump develocity-maven-extension to 2.6.0
(#11405)
add 6772ee33dc3 HDDS-16708. Let dependabot ignore iceberg minor version
upgrades (#11398)
add 3afe9a108f5 HDDS-16673. Avoid redundant ListBuckets RPCs when S3
bucket listing reaches the end (#11397)
add b79d76542ed HDDS-16674. Bump awssdk to 2.55.6 (#11407)
add 4066b0e8e23 HDDS-15991. Speed up deleted table scans in quota repair
(#11386)
add 1ad6848c3c4 HDDS-16241. gRPC deadline kills long-lived block streams
after 30 seconds and the client never recovers (#11080)
add 7ed017d22e4 HDDS-16721. StreamBlockInputStream.read() returns a
negative value for bytes 0x80 to 0xFF (#11411)
add 646c4dfd8c1 HDDS-16643. Remove CleanupTableInfo mechanism (#11365)
add 96c43b3354b HDDS-16362. GetObjectAttributes ObjectParts should return
Part entries for FSO buckets (#11242).
add ccce9187dcb HDDS-15749. Run specific JUnit tests if possible (#10671)
add fb0adda14fc HDDS-16666. Flush SCM transaction in memory during apply
transaction (#11409)
add 7d9e8417e83 HDDS-16008. PutBlocks from Flushes also go without Raft
(#11356)
add a99fabb9dea HDDS-16716. Add description for
ozone.scm.ec.pipeline.per.volume.factor (#11415)
add 77e48039767 HDDS-16682. Allow enabling HTTPS SNI host check via config
(#11389)
add 24cf9aaa28e HDDS-16418. Make Inter-SCM gRPC timeouts and keepalive
configurable (#11295)
add f7b7985243f Handle missing RPC user in S3SecretRequestHelper.
add 1dfb6c982e3 HDDS-16683. Upgrade Jetty from 12.0.38 to 12.1.14 (#11424)
add 402a0622fe5 HDDS-16738. Let dependabot ignore jetty minor version
upgrades (#11422)
add e62603cf3ca HDDS-16611. Use MutableGaugeLong for total count metrics
in OMMetrics (#11410)
add 14083e42662 HDDS-16581. Bump ozone-runner to 20260922-1 (#11431)
add fd640cac6ab Merge remote-tracking branch 'up/master' into HDDS-11233
add 96cb1a6ca40 HDDS-16116. Resolve build failures due to master merge
new 6a58b984299 HDDS-16116. Merge master changes into HDDS-11233
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ci.md | 1 +
.github/dependabot.yml | 23 +-
.github/labeler.yml | 90 +
.github/workflows/asf-allowlist-check.yaml | 2 +-
.github/workflows/build-ratis.yml | 4 +-
.github/workflows/check.yml | 12 +-
.github/workflows/ci-with-ratis.yml | 4 +-
.github/workflows/ci.yml | 81 +-
.github/workflows/intermittent-test-check.yml | 8 +-
.github/workflows/label-pr.yml | 39 +-
.github/workflows/populate-cache.yml | 15 +-
.github/workflows/post-commit.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 4 +-
.github/workflows/schedule-label-pr.yml | 7 +-
.github/workflows/scheduled-cache-update.yml | 2 +-
.github/workflows/zizmor.yml | 4 +-
.gitignore | 1 +
.mvn/extensions.xml | 2 +-
CONTRIBUTING.md | 4 +-
dev-support/ci/errorprone.bats | 102 +
.../ci/junit_summary.bats | 20 +-
dev-support/ci/junit_summary.py | 189 ++
dev-support/ci/selective_ci_checks.bats | 78 +-
dev-support/ci/selective_ci_checks.sh | 88 +-
dev-support/ci/test_junit_summary.py | 226 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 69 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 111 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 84 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 152 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 17 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 177 +-
.../hdds/scm/storage/DomainSocketFactory.java | 4 +
.../hdds/scm/storage/LocalChunkInputStream.java | 48 +-
.../hdds/scm/storage/MultipartInputStream.java | 63 +-
.../hadoop/hdds/scm/storage/PartInputStream.java | 13 +
.../hdds/scm/storage/StreamBlockInputStream.java | 182 +-
.../io/ECBlockReconstructedStripeInputStream.java | 30 +-
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +-
.../hdds/scm/storage/DummyChunkInputStream.java | 22 +-
.../hdds/scm/storage/MockDatanodePipeline.java | 45 +-
.../hdds/scm/storage/PositionedReadTestHelper.java | 92 +
.../scm/storage/TestBlockDataStreamOutput.java | 92 +-
.../hdds/scm/storage/TestBlockInputStream.java | 109 +
.../hdds/scm/storage/TestChunkInputStream.java | 71 +
.../hdds/scm/storage/TestMultipartInputStream.java | 195 ++
.../scm/storage/TestStreamBlockInputStream.java | 361 +++
.../ozone/client/io/TestECBlockInputStream.java | 2 +-
.../TestECBlockReconstructedStripeInputStream.java | 42 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 29 +-
.../hadoop/hdds/conf/DelegatingProperties.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 42 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 34 +
.../hadoop/hdds/scm/StreamingReadResponse.java | 52 +
.../scm/container/ReplicationManagerReport.java | 10 -
.../balancer/ContainerBalancerAdvisor.java | 438 +++
.../balancer/ContainerBalancerClusterAnalyzer.java | 293 ++
.../balancer/ContainerBalancerClusterSnapshot.java | 250 ++
.../balancer/ContainerBalancerConfiguration.java | 17 +-
.../balancer/ContainerBalancerEstimation.java | 205 ++
.../balancer/ContainerBalancerProfile.java | 53 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 39 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 105 +-
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 28 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 4 +
.../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +-
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 9 +
.../apache/hadoop/hdds/utils/db/Proto2Codec.java | 7 +-
.../hadoop/hdds/utils/db/StringCodecBase.java | 9 +-
.../hadoop/io_/retry/RetryInvocationHandler.java | 3 +
.../main/java/org/apache/hadoop/ipc_/Client.java | 33 +-
.../src/main/java/org/apache/hadoop/ipc_/RPC.java | 3 +
.../apache/hadoop/ipc_/RpcInvocationHandler.java | 11 +-
.../java/org/apache/hadoop/ipc_/RpcProxy.java} | 16 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 7 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 68 +
.../apache/hadoop/ozone/common/ChecksumCache.java | 10 +-
.../apache/hadoop/ozone/common/ChunkBuffer.java | 30 +
.../common/ChunkBufferImplWithByteBuffer.java | 12 +
.../common/ChunkBufferImplWithByteBufferList.java | 31 +-
.../ozone/common/IncrementalChunkBuffer.java | 29 +-
.../hadoop/ozone/util/ConcurrentMutableRate.java | 49 +
.../org/apache/hadoop/ozone/util/MetricUtil.java | 6 +-
.../common/src/main/resources/ozone-default.xml | 174 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 16 +
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 83 +
.../hadoop/hdds/scm/TestStreamingReadResponse.java | 172 ++
.../balancer/TestContainerBalancerAdvisor.java | 296 ++
.../TestContainerBalancerClusterAnalyzer.java | 296 ++
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 12 +
.../hdds/utils/TestConnectionFailureUtils.java | 14 +
.../apache/hadoop/hdds/utils/db/CodecTestUtil.java | 6 +
.../hadoop/hdds/utils/db/Proto2CodecTestBase.java | 13 +
.../apache/hadoop/io_/retry/TestRetryProxy.java | 23 +
.../ozone/common/ChunkBufferPutBenchmark.java | 407 +++
.../ozone/common/JfrByteBufferAllocations.java | 133 +
.../apache/hadoop/ozone/common/TestChecksum.java | 16 +
.../hadoop/ozone/common/TestChecksumCache.java | 33 +
.../hadoop/ozone/common/TestChunkBuffer.java | 271 ++
hadoop-hdds/config/pom.xml | 4 +
.../hadoop/hdds/conf/ConfigurationSource.java | 10 +-
.../hdds/conf/MutableConfigurationSource.java | 67 +
.../org/apache/hadoop/hdds/conf/StorageSize.java | 2 +-
.../hadoop/hdds/conf/TestConfigurationSource.java | 8 +
.../apache/hadoop/hdds/conf/TestStorageSize.java | 65 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 26 +-
.../common/impl/BlockDeletingService.java | 13 +-
.../common/impl/ContainerLayoutVersion.java | 22 +-
.../container/common/impl/HddsDispatcher.java | 2 +-
.../ozone/container/common/interfaces/Handler.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 49 +
.../common/statemachine/DatanodeStateMachine.java | 12 +
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../ClosePipelineCommandHandler.java | 4 +-
.../states/datanode/RunningDatanodeState.java | 19 +-
.../states/endpoint/VersionEndpointTask.java | 31 +-
.../transport/server/GrpcXceiverService.java | 68 +-
.../common/transport/server/XceiverServerGrpc.java | 5 +-
.../server/ratis/ContainerStateMachine.java | 31 +-
.../transport/server/ratis/DispatcherContext.java | 4 +-
.../common/transport/server/ratis/LocalStream.java | 22 +-
.../container/common/utils/ContainerCache.java | 4 +-
.../container/common/utils/ReferenceCountedDB.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 90 +-
.../container/keyvalue/ReadBlockComputation.java | 159 ++
.../ozone/container/metadata/DatanodeTable.java | 5 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 116 +-
.../container/replication/ReplicationServer.java | 4 +-
.../checksum/ContainerMerkleTreeTestUtils.java | 34 +-
.../container/checksum/TestContainerDiff.java | 19 +-
.../container/common/TestBlockDeletingService.java | 154 +-
.../container/common/TestDatanodeStateMachine.java | 137 +
.../container/common/impl/TestHddsDispatcher.java | 5 +-
.../states/datanode/TestRunningDatanodeState.java | 127 +-
.../transport/server/TestGrpcXceiverService.java | 169 ++
.../keyvalue/ContainerLayoutTestInfo.java | 90 +-
...ruptions.java => ContainerTestCorruptions.java} | 12 +-
...ava => KeyValueContainerIntegrityTestBase.java} | 4 +-
...stContainerReconciliationWithMockDatanodes.java | 26 +-
.../container/keyvalue/TestKeyValueContainer.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 24 +-
.../TestKeyValueContainerMetadataInspector.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 374 ++-
.../container/ozoneimpl/TestOzoneContainer.java | 81 +
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 6 +
.../docs/content/design/efficient-snapdiff.md | 9 +-
hadoop-hdds/docs/content/design/ozone-sts.md | 56 +-
.../docs/content/feature/Reconfigurability.md | 4 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 3 +-
hadoop-hdds/docs/content/interface/S3.md | 24 +
hadoop-hdds/framework/pom.xml | 24 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 5 +-
.../SecretKeyProtocolClientSideTranslatorPB.java | 2 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 4 +-
...lockLocationProtocolClientSideTranslatorPB.java | 2 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 91 +-
.../certificate/client/SCMCertificateClient.java | 2 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 3 +-
.../hdds/server/http/AdminAuthorizedServlet.java | 2 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 80 +-
...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 258 +-
.../hdds/server/http/ServletElementsFactory.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 14 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 48 +-
.../org/apache/hadoop/hdds/utils/NettyMetrics.java | 108 +-
.../apache/hadoop/hdds/utils/ScmNodeAddress.java | 31 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 35 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 84 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 22 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 150 +
.../utils/db/cache/TableCacheUpdateTracker.java | 88 +
.../exceptions/TestSCMExceptionResultCodes.java | 3 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 19 +
.../TestSCMFailoverProxyProviderChangeConfig.java | 149 +
.../TestSCMSecurityExceptionErrorCodes.java} | 34 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 16 +
.../hdds/server/http/TestBaseHttpServer.java | 83 +
.../hadoop/hdds/server/http/TestHttpServer2.java | 494 ++++
.../hdds/server/http/TestHttpServer2SSL.java | 62 +-
.../apache/hadoop/hdds/utils/TestNettyMetrics.java | 89 +
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 145 +-
.../org/apache/hadoop/hdds/utils/db/TestCodec.java | 14 +
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 46 +
.../apache/hadoop/hdds/utils/db/TestRDBTable.java | 23 +
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 132 +
.../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 +
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +
hadoop-hdds/interface-admin/pom.xml | 1 +
hadoop-hdds/interface-client/pom.xml | 1 +
hadoop-hdds/interface-server/pom.xml | 1 +
hadoop-hdds/rocks-native/pom.xml | 2 +-
...wSstFileRecords.java => RawSstFileRecords.java} | 4 +-
...estLatestVersionedKWayMergeIteratorOverSst.java | 4 +-
.../org/apache/ozone/rocksdiff/CompactionDag.java | 2 +-
.../apache/ozone/rocksdiff/TestCompactionDag.java | 19 +
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 169 +-
.../apache/ozone/rocksdiff/TestRocksDiffUtils.java | 18 +-
hadoop-hdds/server-scm/pom.xml | 12 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 33 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 27 +-
.../scm/container/balancer/ContainerBalancer.java | 18 +-
.../ContainerBalancerSelectionCriteria.java | 22 +-
.../container/balancer/ContainerBalancerTask.java | 48 +-
.../replication/ContainerCheckRequest.java | 11 +
.../container/replication/ReplicationManager.java | 6 +-
.../replication/ReplicationManagerMetrics.java | 37 +-
.../health/ClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/ClosingContainerHandler.java | 2 +-
.../health/ECMisReplicationCheckHandler.java | 4 +-
.../health/ECReplicationCheckHandler.java | 7 +-
.../replication/health/EmptyContainerHandler.java | 6 +-
.../replication/health/OpenContainerHandler.java | 8 +-
.../health/QuasiClosedContainerHandler.java | 2 +-
.../health/QuasiClosedStuckReplicationCheck.java | 9 +-
.../health/RatisReplicationCheckHandler.java | 10 +-
.../RatisUnhealthyReplicationCheckHandler.java | 8 +-
.../health/VulnerableUnhealthyReplicasHandler.java | 4 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 12 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 23 +-
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 6 +
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 51 +
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 15 +
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 8 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 4 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 -
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 15 -
.../hdds/scm/node/NodeDecommissionManager.java | 6 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 32 +-
.../hdds/scm/node/PendingContainerTracker.java | 2 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 244 +-
.../hadoop/hdds/scm/node/states/DatanodeEntry.java | 6 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 5 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 24 +-
.../hdds/scm/server/StorageContainerManager.java | 5 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 39 +
.../org/apache/hadoop/hdds/scm/TestScmUtils.java | 63 +
.../container/balancer/TestContainerBalancer.java | 50 +
.../TestContainerBalancerDatanodeNodeLimit.java | 88 +
.../TestContainerBalancerSelectionCriteria.java | 64 +
.../replication/TestContainerCheckRequest.java | 58 +
.../TestQuasiClosedStuckReplicationCheck.java | 1 -
.../hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java | 119 +
.../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++
.../ha/TestSCMHATransactionBufferMonitorTask.java | 209 +-
.../hadoop/hdds/scm/ha/TestSCMRatisServerImpl.java | 93 +-
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 115 +
.../hdds/scm/ha/TestSequenceIDGenerator.java | 53 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 181 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 85 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 223 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 21 +-
.../TestStorageContainerManagerRatisRoles.java | 126 +
hadoop-hdds/test-utils/pom.xml | 4 +
.../org/apache/ozone/test/GenericTestUtils.java | 7 +-
.../cli/ContainerBalancerAssessmentSubcommand.java | 316 +++
.../hdds/scm/cli/ContainerBalancerCommands.java | 38 +-
...nd.java => ContainerBalancerConfigOptions.java} | 149 +-
.../cli/ContainerBalancerEstimateSubcommand.java | 175 ++
.../scm/cli/ContainerBalancerStartSubcommand.java | 92 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 10 +-
.../admin/om/BucketDeletedBytesSubCommand.java | 101 +
.../ozone/admin/om/GetServiceRolesSubcommand.java | 8 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../admin/scm/TransferScmLeaderSubCommand.java | 2 +-
.../TestContainerBalancerAssessmentSubcommand.java | 176 ++
.../TestContainerBalancerSubCommand.java | 264 +-
.../hdds/scm/cli/TestSafeModeCheckSubcommand.java | 123 +
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 85 +-
.../debug/replicas/BlockExistenceVerifier.java | 2 +-
.../debug/replicas/chunk/ChunkKeyHandler.java | 2 +-
.../java/org/apache/ozone/TestPrintableGraph.java | 11 +-
.../schemaupgrade/UpgradeContainerSchema.java | 4 +-
.../schemaupgrade/TestUpgradeContainerSchema.java | 9 +
.../ozone/shell/s3/RevokeSTSTokenHandler.java | 73 +
.../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 3 +-
.../ozone/shell/snapshot/CountSnapshotHandler.java | 91 +
.../ozone/shell/snapshot/SnapshotCommands.java | 1 +
.../shell/snapshot/TestCountSnapshotHandler.java | 122 +
.../apache/hadoop/ozone/client/ObjectStore.java | 47 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 51 +-
.../ozone/client/S3HeadObjectAttributes.java | 55 +
.../client/checksum/ECBlockChecksumComputer.java | 2 +-
.../client/checksum/ECFileChecksumHelper.java | 2 +-
.../checksum/ReplicatedFileChecksumHelper.java | 2 +-
.../client/io/BlockDataStreamOutputEntryPool.java | 8 +-
.../client/io/BlockOutputStreamEntryPool.java | 14 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 5 +-
.../ozone/client/io/KeyDataStreamOutput.java | 4 +
.../hadoop/ozone/client/io/KeyOutputStream.java | 4 +
.../ozone/client/io/OzoneCryptoInputStream.java | 164 +-
.../ozone/client/io/OzoneDataStreamOutput.java | 10 +-
.../hadoop/ozone/client/io/OzoneOutputStream.java | 10 +-
.../ozone/client/protocol/ClientProtocol.java | 111 +-
.../client/protocol/ListStatusLightOptions.java | 182 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 123 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 21 +
.../checksum/TestECBlockChecksumComputer.java | 122 +
.../ozone/client/io/TestKeyDataStreamOutput.java | 2 +-
.../ozone/client/io/TestKeyOutputStream.java | 76 +
.../client/io/TestOzoneCryptoInputStream.java | 504 ++++
.../hadoop/ozone/client/rpc/TestRpcClient.java | 17 +
hadoop-ozone/common/pom.xml | 4 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 30 +-
.../ozone/conf/S3GatewayHealthCheckConfig.java | 95 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 39 +-
.../hadoop/ozone/om/exceptions/OMException.java | 4 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 245 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 +
.../ozone/om/helpers/AssumeRoleResponseInfo.java | 109 +
.../ozone/om/helpers/AwsRoleArnValidator.java | 146 +
.../ozone/om/helpers/BucketDeletedBytes.java | 79 +
.../ozone/om/helpers/CallerIdentityInfo.java | 88 +
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 11 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 19 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 13 +
.../helpers/OmMultipartCommitUploadPartInfo.java | 9 +-
.../hadoop/ozone/om/helpers/OmRangerSyncArgs.java | 3 -
.../apache/hadoop/ozone/om/helpers/S3STSUtils.java | 226 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 64 +-
.../apache/hadoop/ozone/om/protocol/S3Auth.java | 20 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 57 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +-
.../om/protocolPB/OzoneManagerClientProtocol.java | 10 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 212 +-
.../om/protocolPB/OzoneManagerProtocolPB.java | 3 +-
.../security/acl/iam/IamSessionPolicyResolver.java | 1231 ++++++++
.../ozone/security/acl/iam/package-info.java | 8 +-
.../SnapshotBucketCount.java} | 41 +-
.../SnapshotCountResponse.java} | 37 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 80 +
.../ozone/om/exceptions/TestResultCodes.java | 2 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 236 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 17 +
.../om/helpers/TestAssumeRoleResponseInfo.java | 197 ++
.../ozone/om/helpers/TestAwsRoleArnValidator.java | 133 +
.../ozone/om/helpers/TestCallerIdentityInfo.java | 107 +
.../hadoop/ozone/om/helpers/TestOMRatisHelper.java | 106 +
.../hadoop/ozone/om/helpers/TestS3STSUtils.java | 98 +
.../om/helpers/TestS3STSUtilsCallerIdentity.java | 59 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 171 ++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 +
.../grpc/TestClientAddressClientInterceptor.java | 1 +
.../grpc/TestClientAddressServerInterceptor.java | 14 +-
.../acl/iam/TestIamSessionPolicyResolver.java | 2927 ++++++++++++++++++++
.../ozone/security/acl/iam/package-info.java | 8 +-
hadoop-ozone/dev-support/checks/bats.sh | 2 +
hadoop-ozone/dev-support/checks/errorprone.sh | 62 +
hadoop-ozone/dev-support/checks/findbugs.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 26 +-
hadoop-ozone/dev-support/checks/junit.sh | 3 +-
hadoop-ozone/dev-support/checks/license.sh | 1 +
hadoop-ozone/dist/pom.xml | 22 +-
.../Ozone - Memory Consumption Metrics.json | 4 +-
.../dist/src/main/compose/common/init-kdc.sh | 3 +
.../dist/src/main/compose/common/ranger.yaml | 16 +-
.../dist/src/main/compose/common/s3-haproxy.cfg | 11 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 3 +
.../main/compose/ozonesecure-ha/polaris-setup.sh | 121 +
.../compose/ozonesecure-ha/polaris-smoketest.sh | 112 +
.../src/main/compose/ozonesecure-ha/polaris.yaml | 77 +
.../main/compose/ozonesecure-ha/ranger-testlib.sh | 111 +
.../src/main/compose/ozonesecure-ha/ranger.yaml | 6 +-
.../src/main/compose/ozonesecure-ha/s3-haproxy.cfg | 11 +-
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 50 +-
.../compose/ozonesecure-ha/test-repair-tools.sh | 67 +-
.../compose/ozonesecure-ha/{.env => test-sts.sh} | 28 +-
.../src/main/compose/ozonesecure/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +-
.../dist/src/main/compose/upgrade/testlib.sh | 3 +-
.../dist/src/main/compose/xcompat/clients.yaml | 8 +
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +-
.../src/main/k8s/definitions/ozone/s3g-ss.yaml | 6 +-
.../examples/getting-started/s3g-statefulset.yaml | 6 +-
.../k8s/examples/minikube/s3g-statefulset.yaml | 6 +-
.../k8s/examples/ozone-dev/s3g-statefulset.yaml | 6 +-
.../k8s/examples/ozone-ha/s3g-statefulset.yaml | 6 +-
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 6 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 24 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 20 +-
.../smoketest/repair/om-update-transaction.robot | 56 +
.../dist/src/main/smoketest/s3/bucketcreate.robot | 55 +
.../dist/src/main/smoketest/s3/health.robot | 43 +
.../src/main/smoketest/s3/secretgenerate.robot | 3 +-
.../security/generate_oversized_session_policy.py} | 34 +-
.../smoketest/security/mutate_sts_session_token.py | 216 ++
.../security/ozone-secure-sts-multitenant.robot | 336 +++
.../security/ozone-secure-sts-polaris.sql | 27 +
.../smoketest/security/ozone-secure-sts.resource | 376 +++
.../main/smoketest/security/ozone-secure-sts.robot | 1474 ++++++++++
.../smoketest/security/ozone-secure-token.robot | 11 +
hadoop-ozone/freon/pom.xml | 1 +
.../ozone/freon/HadoopFsReadWriteValidator.java | 263 ++
hadoop-ozone/httpfsgateway/pom.xml | 12 +-
.../fs/http/server/HttpFSExceptionProvider.java | 5 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 2 +-
.../fs/http/server/HttpFSServerWebServer.java | 4 +
.../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +-
.../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +-
.../http/server/TestHttpFSExceptionProvider.java | 59 +
hadoop-ozone/iceberg/pom.xml | 26 +-
hadoop-ozone/insight/pom.xml | 3 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +-
.../src/test/resources/ozone-site.xml | 77 +-
hadoop-ozone/integration-test-s3/pom.xml | 40 +-
.../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +-
.../org/apache/hadoop/ozone/s3/ProxyServer.java | 50 +-
.../ozone/s3/ProxyServerIntegrationTest.java | 4 +-
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 +
.../hadoop/ozone/s3/TestS3GatewayHealthCheck.java | 126 +
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 33 +
.../hadoop/ozone/s3/awssdk/package-info.java | 8 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 222 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 689 ++++-
.../hadoop/ozone/s3/awssdk/v2/package-info.java | 8 +-
.../src/test/resources/ozone-site.xml | 86 +-
hadoop-ozone/integration-test/pom.xml | 50 +-
.../fs/contract/AbstractContractSafeModeTest.java | 2 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 66 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 26 +-
.../fs/ozone/TestOfsGetFileStatusBenchmark.java | 356 +++
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 8 +-
.../hadoop/hdds/scm/TestContainerOperations.java | 25 +-
.../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++
.../hadoop/hdds/scm/TestRackAwarePlacement.java | 4 +
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 104 +-
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 113 +-
.../hdds/scm/TestStorageContainerManager.java | 24 +-
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 54 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 75 +-
.../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 4 +-
.../hadoop/hdds/scm/TestXceiverClientMetrics.java | 118 +-
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 +
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 2 +-
.../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 122 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 139 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 116 +-
.../client/rpc/TestBlockDataStreamOutput.java | 42 +-
...stClientRetryContainerStateMachineFailures.java | 5 +-
.../client/rpc/TestContainerStateMachine.java | 5 +-
.../rpc/TestContainerStateMachineFailures.java | 5 +-
.../rpc/TestContainerStateMachineStream.java | 10 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 5 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 127 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 2 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 5 +-
.../ozone/client/rpc/read/InputStreamTests.java | 43 +-
.../client/rpc/read/TestChunkInputStream.java | 71 +-
.../client/rpc/read/TestLocalChunkInputStream.java | 100 +-
.../rpc/read/TestStreamBlockInputStream.java | 145 +-
.../ozone/client/rpc/read/TestStreamRead.java | 179 +-
.../commandhandler/TestBlockDeletion.java | 4 +-
.../TestContainerCommandReconciliation.java | 4 +-
...tBackgroundContainerDataScannerIntegration.java | 12 +-
...kgroundContainerMetadataScannerIntegration.java | 18 +-
.../TestOnDemandContainerScannerIntegration.java | 32 +-
.../freon/TestHadoopFsReadWriteValidator.java | 229 ++
.../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 2 +-
.../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 53 +-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 7 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 8 +-
.../TestDirectoryDeletingServiceWithFSO.java | 97 +-
.../TestOmMixedWorkloadUnderDeletionBench.java | 715 +++++
...TestSnapshotDeletingServiceIntegrationTest.java | 131 +-
.../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 2 +
.../snapshot/TestOmSnapshotDefragSpaceSavings.java | 713 +++++
.../om/snapshot/TestOzoneManagerHASnapshot.java | 32 +-
.../TestSnapshotDirectoryCleaningService.java | 15 +-
.../reconfig/TestDatanodeReconfiguration.java | 2 -
.../hadoop/ozone/shell/TestOzoneRepairShell.java | 3 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 56 +-
.../ozone/shell/TestTransferLeadershipShell.java | 60 +-
.../tools/contract/AbstractContractDistCpTest.java | 2 +-
.../org/apache/ozone/test/ClusterForTests.java | 5 +-
.../java/org/apache/ozone/test/FreonTests.java | 9 +
.../java/org/apache/ozone/test/KerberosTests.java | 191 ++
.../src/test/resources/ozone-site.xml | 90 +-
hadoop-ozone/interface-client/pom.xml | 1 +
.../src/main/proto/OmClientProtocol.proto | 153 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 11 +-
.../ozone/om/helpers/TestTransactionInfoCodec.java | 11 +
hadoop-ozone/mini-cluster/pom.xml | 9 +
.../hadoop/ozone/ClientConfigForTesting.java | 54 +-
.../org/apache/hadoop/ozone/FixedHostMapping.java | 153 -
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 34 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 43 +-
hadoop-ozone/ozone-manager/pom.xml | 16 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 40 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 74 +-
.../org/apache/hadoop/ozone/om/ListIterator.java | 7 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 38 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 403 +--
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 167 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 604 +++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 34 +-
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 34 +
.../apache/hadoop/ozone/om/S3SecretManager.java | 4 +
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 81 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 163 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 16 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 24 +-
.../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 2 +
.../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +-
.../hadoop/ozone/om/ratis/OMThreadContext.java | 198 ++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 87 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 30 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 14 +
.../hadoop/ozone/om/request/OMClientRequest.java | 93 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 7 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 9 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 11 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 11 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 9 +-
.../om/request/file/OMDirectoryCreateRequest.java | 7 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 12 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../request/file/OMFileCreateRequestWithFSO.java | 5 +-
.../om/request/file/OMRecoverLeaseRequest.java | 2 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 557 +++-
.../ozone/om/request/key/OMKeyCommitRequest.java | 12 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 39 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 8 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 7 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 8 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 29 +-
.../ozone/om/request/key/OMKeySetTimesRequest.java | 2 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 15 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 11 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 11 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 11 +-
.../OMLifecycleConfigurationDeleteRequest.java | 2 +-
.../OMLifecycleConfigurationSetRequest.java | 2 +-
.../lifecycle/OMLifecycleSaveScanStateRequest.java | 2 +-
.../OMLifecycleSetServiceStatusRequest.java | 2 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 333 +--
.../S3InitiateMultipartUploadRequestWithFSO.java | 5 +-
.../multipart/S3MultipartUploadAbortRequest.java | 7 +-
.../S3MultipartUploadCommitPartRequest.java | 7 +-
.../S3MultipartUploadCompleteRequest.java | 26 +-
.../om/request/s3/security/OMSetSecretRequest.java | 8 +-
.../request/s3/security/S3AssumeRoleRequest.java | 611 ++++
.../security/S3DeleteRevokedSTSTokensRequest.java} | 55 +-
.../om/request/s3/security/S3GetSecretRequest.java | 23 +-
.../s3/security/S3RevokeSTSTokenRequest.java | 157 ++
.../request/s3/security/S3RevokeSecretRequest.java | 16 +-
.../request/s3/security/S3SecretRequestHelper.java | 8 +-
.../s3/tagging/S3BucketTaggingRequestBase.java | 2 +-
.../security/OMCancelDelegationTokenRequest.java | 2 +-
.../security/OMGetDelegationTokenRequest.java | 2 +-
.../security/OMRenewDelegationTokenRequest.java | 2 +-
.../snapshot/OMSnapshotMoveTableKeysRequest.java | 7 +-
.../om/request/validation/ValidationCondition.java | 21 +-
.../om/request/validation/VersionExtractor.java | 45 +-
.../om/request/volume/OMQuotaRepairRequest.java | 11 +
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 10 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 10 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 6 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 8 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../hadoop/ozone/om/response/CleanupTableInfo.java | 51 -
.../ozone/om/response/DummyOMClientResponse.java | 1 -
.../hadoop/ozone/om/response/OMClientResponse.java | 24 +-
.../om/response/bucket/OMBucketCreateResponse.java | 5 -
.../om/response/bucket/OMBucketDeleteResponse.java | 6 -
.../response/bucket/OMBucketSetOwnerResponse.java | 4 -
.../bucket/OMBucketSetPropertyResponse.java | 4 -
.../response/bucket/acl/OMBucketAclResponse.java | 4 -
.../response/file/OMDirectoryCreateResponse.java | 4 -
.../file/OMDirectoryCreateResponseWithFSO.java | 14 +-
.../om/response/file/OMFileCreateResponse.java | 5 -
.../response/file/OMFileCreateResponseWithFSO.java | 7 -
.../om/response/file/OMRecoverLeaseResponse.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 5 -
.../key/OMAllocateBlockResponseWithFSO.java | 5 -
.../key/OMDirectoriesPurgeResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 8 -
.../response/key/OMKeyCommitResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 -
.../response/key/OMKeyCreateResponseWithFSO.java | 7 -
.../ozone/om/response/key/OMKeyDeleteResponse.java | 7 -
.../response/key/OMKeyDeleteResponseWithFSO.java | 10 -
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 -
.../ozone/om/response/key/OMKeyRenameResponse.java | 5 -
.../response/key/OMKeyRenameResponseWithFSO.java | 7 -
.../om/response/key/OMKeySetTimesResponse.java | 4 -
.../response/key/OMKeySetTimesResponseWithFSO.java | 5 -
.../om/response/key/OMKeysDeleteResponse.java | 7 -
.../response/key/OMKeysDeleteResponseWithFSO.java | 11 -
.../om/response/key/OMKeysRenameResponse.java | 4 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 8 -
.../om/response/key/acl/OMKeyAclResponse.java | 4 -
.../response/key/acl/OMKeyAclResponseWithFSO.java | 5 -
.../key/acl/prefix/OMPrefixAclResponse.java | 4 -
.../OMLifecycleConfigurationDeleteResponse.java | 4 -
.../OMLifecycleConfigurationSetResponse.java | 4 -
.../OMLifecycleSaveScanStateResponse.java | 3 -
.../OMLifecycleSetServiceStatusResponse.java | 2 -
.../S3ExpiredMultipartUploadsAbortResponse.java | 10 -
.../S3InitiateMultipartUploadResponse.java | 5 -
.../S3InitiateMultipartUploadResponseWithFSO.java | 7 -
.../multipart/S3MultipartUploadAbortResponse.java | 9 -
.../S3MultipartUploadAbortResponseWithFSO.java | 9 -
.../S3MultipartUploadCommitPartResponse.java | 8 -
...S3MultipartUploadCommitPartResponseWithFSO.java | 9 -
.../S3MultipartUploadCompleteResponse.java | 10 -
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 -
.../response/s3/security/OMSetSecretResponse.java | 4 -
.../security/S3AssumeRoleResponse.java} | 18 +-
.../S3DeleteRevokedSTSTokensResponse.java} | 38 +-
.../response/s3/security/S3GetSecretResponse.java | 4 -
.../security/S3RevokeSTSTokenResponse.java} | 30 +-
.../s3/security/S3RevokeSecretResponse.java | 4 -
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 -
.../S3DeleteObjectTaggingResponseWithFSO.java | 4 -
.../s3/tagging/S3PutObjectTaggingResponse.java | 4 -
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 -
.../tenant/OMSetRangerServiceVersionResponse.java | 4 -
.../s3/tenant/OMTenantAssignAdminResponse.java | 6 -
.../tenant/OMTenantAssignUserAccessIdResponse.java | 10 -
.../response/s3/tenant/OMTenantCreateResponse.java | 8 -
.../response/s3/tenant/OMTenantDeleteResponse.java | 8 -
.../s3/tenant/OMTenantRevokeAdminResponse.java | 6 -
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 -
.../security/OMCancelDelegationTokenResponse.java | 4 -
.../security/OMGetDelegationTokenResponse.java | 4 -
.../security/OMRenewDelegationTokenResponse.java | 4 -
.../snapshot/OMSnapshotCreateResponse.java | 7 -
.../snapshot/OMSnapshotDeleteResponse.java | 4 -
.../OMSnapshotMoveDeletedKeysResponse.java | 3 -
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 -
.../response/snapshot/OMSnapshotPurgeResponse.java | 8 -
.../snapshot/OMSnapshotRenameResponse.java | 4 -
.../snapshot/OMSnapshotSetPropertyResponse.java | 4 -
.../response/upgrade/OMCancelPrepareResponse.java | 4 -
.../upgrade/OMFinalizeUpgradeResponse.java | 3 -
.../om/response/upgrade/OMPrepareResponse.java | 3 -
.../om/response/util/OMEchoRPCWriteResponse.java | 2 -
.../om/response/volume/OMQuotaRepairResponse.java | 5 -
.../om/response/volume/OMVolumeAclOpResponse.java | 4 -
.../om/response/volume/OMVolumeCreateResponse.java | 4 -
.../om/response/volume/OMVolumeDeleteResponse.java | 4 -
.../response/volume/OMVolumeSetOwnerResponse.java | 4 -
.../response/volume/OMVolumeSetQuotaResponse.java | 4 -
.../ozone/om/service/DirectoryDeletingService.java | 58 +-
.../ozone/om/service/KeyLifecycleService.java | 12 +-
.../om/service/KeyLifecycleServiceMetrics.java | 31 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 256 +-
.../om/service/RevokedSTSTokenCleanupService.java | 269 ++
.../hadoop/ozone/om/snapshot/ReferenceCounted.java | 4 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 22 +-
.../hadoop/ozone/om/snapshot/diff/EntryValue.java | 150 +
.../om/snapshot/diff/FullDiffSequentialReader.java | 205 ++
.../ozone/om/snapshot/diff/SnapDiffJobStore.java | 370 +++
.../{ => diff}/SnapshotDiffValueParser.java | 35 +-
.../trapped/BucketDeletedDataCalculator.java | 373 +++
.../ozone/om/snapshot/trapped/package-info.java | 8 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 47 +-
.../protocolPB/OzoneManagerRequestHandler.java | 89 +-
.../hadoop/ozone/security/S3SecurityUtil.java | 124 +
.../hadoop/ozone/security/STSSecurityUtil.java | 344 +++
.../hadoop/ozone/security/STSTokenEncryption.java | 205 ++
.../hadoop/ozone/security/STSTokenIdentifier.java | 482 ++++
.../ozone/security/STSTokenSecretManager.java | 234 ++
.../apache/hadoop/ozone/om/FaultInjectorImpl.java | 12 +
.../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 +
.../ozone/om/TestGrpcOzoneManagerServer.java | 152 +
.../hadoop/ozone/om/TestOMMetadataReader.java | 724 ++++-
...anagerServer.java => TestOMMetricsSetters.java} | 48 +-
.../hadoop/ozone/om/TestOMPerformanceMetrics.java | 107 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 43 +
.../ozone/om/TestOmSnapshotLeakDetection.java | 87 +
.../hadoop/ozone/om/TestOzoneManagerS3Auth.java | 123 +
.../ozone/om/TestOzoneManagerSnapshotCount.java | 149 +
.../org/apache/hadoop/ozone/om/TestScmClient.java | 24 +
.../om/TestTrashOzoneFileSystemLegacyBucket.java | 276 ++
.../ozone/om/execution/TestOMExecutionFlow.java | 165 ++
.../ozone/om/lock/TestOMLockDetailsUtil.java | 78 +
.../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++
...estOzoneManagerDoubleBufferTransactionInfo.java | 235 ++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 58 +
.../om/ratis/TestOzoneManagerRatisServer.java | 50 +
.../om/ratis/TestOzoneManagerStateMachine.java | 493 +++-
.../apache/hadoop/ozone/om/ratis/package-info.java | 8 +-
.../request/TestOMClientRequestWithUserInfo.java | 335 ++-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 28 +
.../key/TestOMDirectoriesPurgeApplyPerf.java | 226 ++
.../TestOMDirectoriesPurgeRequestAndResponse.java | 86 +
.../om/request/key/TestOMKeyCommitRequest.java | 34 +
.../om/request/key/TestOMKeyCreateRequest.java | 102 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 8 +
.../key/TestOMKeyPurgeRequestAndResponse.java | 67 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 48 +
.../key/TestOMKeysDeleteRequestWithFSO.java | 16 +
.../request/key/TestOMOpenKeysDeleteRequest.java | 7 +
...> OMLifecycleConfigurationRequestTestBase.java} | 2 +-
.../TestOMLifecycleConfigurationDeleteRequest.java | 2 +-
.../TestOMLifecycleConfigurationSetRequest.java | 2 +-
.../TestS3ExpiredMultipartUploadsAbortRequest.java | 141 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 5 +
.../TestS3MultipartUploadAbortRequest.java | 10 +
.../TestS3MultipartUploadCompleteRequest.java | 149 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 48 +
.../request/s3/security/S3SecurityTestUtils.java} | 37 +-
.../s3/security/TestS3AssumeRoleRequest.java | 988 +++++++
.../s3/security/TestS3GetSecretRequest.java | 34 +
.../s3/security/TestS3RevokeSTSTokenRequest.java | 402 +++
.../s3/security/TestS3SecretRequestHelper.java | 31 +-
.../ozone/om/request/s3/security/package-info.java | 8 +-
.../request/volume/TestOMQuotaRepairRequest.java | 55 +
.../ozone/om/response/TestCleanupTableInfo.java | 380 ---
.../ozone/om/response/TestOMClientResponse.java} | 41 +-
.../hadoop/ozone/om/response/TestOMResponse.java | 51 +-
.../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 ++
.../file/TestOMDirectoryCreateResponseWithFSO.java | 2 -
.../s3/security/TestS3AssumeRoleResponse.java | 133 +
.../om/service/TestDirectoryDeletingService.java | 60 +-
.../ozone/om/service/TestKeyDeletingService.java | 39 +
.../ozone/om/service/TestKeyLifecycleService.java | 414 ++-
.../ozone/om/service/TestQuotaRepairTask.java | 538 ++++
.../service/TestRevokedSTSTokenCleanupService.java | 449 +++
.../hadoop/ozone/om/service/package-info.java | 8 +-
.../snapshot/TestOmSnapshotLocalDataManager.java | 3 +-
.../ozone/om/snapshot/TestSnapshotChain.java | 45 +
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 +
.../diff/TestFullDiffSequentialReader.java | 320 +++
.../{ => diff}/TestSnapshotDiffValueParser.java | 2 +-
.../trapped/TestBucketDeletedDataCalculator.java | 243 ++
.../protocolPB/TestOzoneManagerRequestHandler.java | 4 +-
.../hadoop/ozone/security/TestS3SecurityUtil.java | 405 +++
.../hadoop/ozone/security/TestSTSSecurityUtil.java | 608 ++++
.../ozone/security/TestSTSTokenEncryption.java | 215 ++
.../ozone/security/TestSTSTokenIdentifier.java | 633 +++++
.../ozone/security/TestSTSTokenSecretManager.java | 213 ++
.../hadoop/ozone/security/acl/package-info.java | 8 +-
hadoop-ozone/ozonefs-common/pom.xml | 7 +
...sDtFetcher.java => AbstractOzoneDtFetcher.java} | 41 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 44 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 15 +-
.../org/apache/hadoop/fs/ozone/O3DtFetcher.java | 48 +
.../org/apache/hadoop/fs/ozone/O3fsDtFetcher.java | 48 +-
.../org/apache/hadoop/fs/ozone/OfsDtFetcher.java | 14 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 179 +-
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 105 +
.../TestBasicRootedOzoneClientAdapterHeadOp.java | 49 +-
.../apache/hadoop/fs/ozone/TestOzoneDtFetcher.java | 140 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 299 ++
hadoop-ozone/ozonefs-hadoop2/pom.xml | 3 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 +
.../org.apache.hadoop.security.token.DtFetcher | 2 +
hadoop-ozone/ozonefs-shaded/pom.xml | 1 +
.../org.apache.hadoop.security.token.DtFetcher | 2 +
.../hadoop/fs/ozone/TestDtFetcherProviders.java} | 34 +-
hadoop-ozone/pom.xml | 18 +-
hadoop-ozone/recon-codegen/pom.xml | 14 +-
hadoop-ozone/recon/pom.xml | 12 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 2 +-
.../hadoop/ozone/recon/api/BucketEndpoint.java | 1 -
.../ozone/recon/api/ClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/UtilizationEndpoint.java | 12 +-
.../hadoop/ozone/recon/api/VolumeEndpoint.java | 1 -
.../ozone/recon/api/filters/ReconAuthFilter.java | 2 +-
.../hadoop/ozone/recon/api/types/EntityType.java | 87 +-
.../ozone/recon/api/types/FeatureProvider.java | 10 +-
.../ozone/recon/fsck/ReconReplicationManager.java | 36 +-
.../ozone/recon/scm/ReconContainerManager.java | 8 +-
.../impl/ReconContainerMetadataManagerImpl.java | 7 -
.../recon/tasks/MultipartInfoInsightHandler.java | 118 +-
.../ozone/recon/tasks/OmTableInsightTask.java | 32 +-
.../recon/upgrade/ReconLayoutVersionManager.java | 95 +-
.../upgrade/ReconTaskStatusTableUpgradeAction.java | 87 +-
.../webapps/recon/ozone-recon-web/package.json | 4 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +-
.../src/v2/components/navBar/navBar.tsx | 12 +-
.../src/v2/pages/heatmap/heatmap.tsx | 35 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 49 +-
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 40 +-
.../recon/api/types/TestFeatureProvider.java} | 32 +-
.../recon/fsck/TestReconReplicationManager.java | 50 +
.../persistence/TestStatsSchemaDefinition.java | 22 +-
.../TestUtilizationSchemaDefinition.java | 18 +-
.../ozone/recon/scm/TestReconContainerManager.java | 7 +-
.../TestStorageContainerServiceProviderImpl.java | 53 +
.../recon/tasks/TestContainerKeyMapperTask.java | 21 +-
.../recon/tasks/TestContainerSizeCountTask.java | 2 +-
.../ozone/recon/tasks/TestOmTableInsightTask.java | 118 +-
.../upgrade/TestReconLayoutVersionManager.java | 23 +-
.../TestReconTaskStatusTableUpgradeAction.java | 176 ++
hadoop-ozone/s3gateway/pom.xml | 30 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 11 +-
.../hadoop/ozone/s3/EmptyContentTypeFilter.java | 30 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../hadoop/ozone/s3/MultiDigestInputStream.java | 8 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 3 +-
.../hadoop/ozone/s3/ReadConsistencyFilter.java | 53 +
.../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 +
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 +
.../hadoop/ozone/s3/S3GatewayReadinessProbe.java | 164 ++
.../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 90 +-
...Mapper.java => S3STSHeadersResponseFilter.java} | 37 +-
.../apache/hadoop/ozone/s3/S3STSHttpServer.java | 108 +
.../hadoop/ozone/s3/SignedChunksInputStream.java | 403 ++-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 46 +-
.../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 +
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 167 +-
.../s3/endpoint/BucketGetLocationHandler.java | 5 +
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 14 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 21 +-
.../BucketOperationNotImplementedHandler.java | 64 +
.../hadoop/ozone/s3/endpoint/CopyPartResult.java | 4 +-
.../hadoop/ozone/s3/endpoint/CopyResult.java} | 25 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 217 +-
.../s3/endpoint/GetObjectAttributesResponse.java | 10 +-
.../s3/endpoint/ListMultipartUploadsHandler.java | 15 +-
.../ozone/s3/endpoint/MultipartKeyHandler.java | 14 +-
.../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +-
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 187 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 264 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 72 +-
.../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 +
.../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +-
.../ObjectOperationNotImplementedHandler.java | 65 +
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 23 +-
.../s3/endpoint/S3LifecycleConfiguration.java | 33 +-
.../s3/endpoint/S3ObjectStreamingWriteGuard.java | 5 +
.../ozone/s3/endpoint/S3ObjectWriteGuard.java | 30 +
.../hadoop/ozone/s3/endpoint/S3RequestContext.java | 14 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 50 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 18 +-
.../hadoop/ozone/s3/exception/OSTSException.java | 173 ++
...ceptionMapper.java => OSTSExceptionMapper.java} | 11 +-
.../s3/exception/OSTSNotFoundExceptionMapper.java | 93 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 60 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 118 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 44 +-
.../s3/signature/AuthorizationV4QueryParser.java | 9 +-
.../hadoop/ozone/s3/signature/ChunksValidator.java | 151 +
.../hadoop/ozone/s3/signature/Credential.java | 20 +
.../s3/signature/MalformedResourceException.java | 19 +-
.../hadoop/ozone/s3/signature/SignatureInfo.java | 61 +-
.../ozone/s3/signature/SignatureProcessor.java | 4 +-
.../ozone/s3/signature/StringToSignProducer.java | 101 +-
.../ozone/s3/util/RangeHeaderParserUtil.java | 6 +-
.../ozone/s3/util/ReadConsistencyContext.java | 112 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 54 +-
.../hadoop/ozone/s3/util/S3GActionIamMapper.java | 95 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 2 +
.../ozone/s3secret/S3SecretEndpointBase.java | 32 +
.../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +-
.../apache/hadoop/ozone/s3sts/Application.java} | 29 +-
.../ozone/s3sts/S3AssumeRoleResponseXml.java | 162 ++
.../s3sts/S3GetCallerIdentityResponseXml.java | 92 +
.../apache/hadoop/ozone/s3sts/S3STSConfigKeys.java | 54 +
.../apache/hadoop/ozone/s3sts/S3STSEnabled.java | 18 +-
.../s3sts/S3STSEnabledEndpointRequestFilter.java | 53 +
.../apache/hadoop/ozone/s3sts/S3STSEndpoint.java | 521 ++++
.../S3STSEndpointBase.java} | 72 +-
.../hadoop/ozone/s3sts/S3STSResponseMetadata.java} | 32 +-
.../apache/hadoop/ozone/s3sts/package-info.java | 19 +-
.../webapps/{s3gateway => s3g-sts}/WEB-INF/web.xml | 17 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 +
.../hadoop/ozone/client/ClientProtocolStub.java | 133 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 227 +-
.../ozone/client/OzoneDataStreamOutputStub.java | 18 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 20 +-
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 120 +-
.../ozone/s3/TestEmptyContentTypeFilter.java | 35 +
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 +
.../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 +
.../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 +
.../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 +
.../ozone/s3/TestS3GatewayLivenessServlet.java | 126 +
.../ozone/s3/TestSignedChunksInputStream.java | 354 ++-
.../ozone/s3/TestVirtualHostStyleFilter.java | 61 +-
.../ozone/s3/endpoint/EndpointTestUtils.java | 40 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++-
.../s3/endpoint/TestBucketNotImplemented.java | 102 +
.../ozone/s3/endpoint/TestCopyActionsAudit.java | 133 +
...estCopyObjectAndUploadPartCopyLastModified.java | 150 +
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 264 ++
.../s3/endpoint/TestListMultipartUploads.java | 166 ++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +
.../s3/endpoint/TestMultipartUploadWithCopy.java | 6 +-
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 273 ++
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 108 +
.../s3/endpoint/TestObjectNotImplemented.java | 113 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 395 ++-
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +-
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 108 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 82 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 111 +-
.../TestS3ActionOverrideForOwnerVerification.java | 174 ++
.../TestS3LifecycleConfigurationDelete.java | 125 +
.../endpoint/TestS3LifecycleConfigurationGet.java | 90 +
.../endpoint/TestS3LifecycleConfigurationPut.java | 184 +-
.../ozone/s3/endpoint/TestS3ObjectWriteGuard.java | 31 +
.../ozone/s3/endpoint/TestS3RequestContext.java | 86 +
.../ozone/s3/endpoint/TestUploadWithStream.java | 137 +-
.../ozone/s3/exception/TestOS3Exceptions.java | 61 +-
.../ozone/s3/exception/TestOSTSExceptions.java | 104 +
.../exception/TestOSTSNotFoundExceptionMapper.java | 120 +
.../ozone/s3/signature/SignatureTestUtils.java | 162 ++
.../s3/signature/TestAWSSignatureProcessor.java | 14 +
.../signature/TestAuthorizationV4HeaderParser.java | 133 +-
.../signature/TestAuthorizationV4QueryParser.java | 55 +-
.../ozone/s3/signature/TestChunksValidator.java | 194 ++
.../s3/signature/TestStringToSignProducer.java | 114 +-
.../ozone/s3/util/TestRangeHeaderParserUtil.java | 7 +
.../ozone/s3/util/TestS3GActionIamMapper.java | 71 +
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 9 +
.../apache/hadoop/ozone/s3/util/package-info.java | 8 +-
.../ozone/s3secret/TestS3SecretEndpointBase.java | 69 +
.../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +-
.../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 +
.../hadoop/ozone/s3sts/TestS3STSEndpoint.java | 994 +++++++
.../apache/hadoop/ozone/s3sts/package-info.java | 8 +-
hadoop-ozone/tools/pom.xml | 5 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 303 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 76 +-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 301 +-
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 126 +-
pom.xml | 179 +-
966 files changed, 62588 insertions(+), 8251 deletions(-)
create mode 100644 .github/labeler.yml
create mode 100644 dev-support/ci/errorprone.bats
copy .github/workflows/scheduled-cache-update.yml =>
dev-support/ci/junit_summary.bats (78%)
create mode 100755 dev-support/ci/junit_summary.py
create mode 100644 dev-support/ci/test_junit_summary.py
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/PositionedReadTestHelper.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestMultipartInputStream.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterAnalyzer.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterSnapshot.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java
copy
hadoop-hdds/{config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=> common/src/main/java/org/apache/hadoop/ipc_/RpcProxy.java} (64%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/ConcurrentMutableRate.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestStreamingReadResponse.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerClusterAnalyzer.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/ChunkBufferPutBenchmark.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/JfrByteBufferAllocations.java
create mode 100644
hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestStorageSize.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/ReadBlockComputation.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/TestGrpcXceiverService.java
rename
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/{TestContainerCorruptions.java
=> ContainerTestCorruptions.java} (95%)
rename
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/{TestKeyValueContainerIntegrityChecks.java
=> KeyValueContainerIntegrityTestBase.java} (98%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/{AdminAuthorizedServlet.java
=> HddsJMXJsonServlet.java} (52%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ScmNodeAddress.java
(55%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCacheUpdateTracker.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/proxy/TestSCMFailoverProxyProviderChangeConfig.java
copy
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/{scm/exceptions/TestSCMExceptionResultCodes.java
=> security/exception/TestSCMSecurityExceptionErrorCodes.java} (50%)
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestNettyMetrics.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTableCacheUpdateTracker.java
rename
hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/{TestRawSstFileRecords.java
=> RawSstFileRecords.java} (98%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestScmUtils.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerCheckRequest.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHADBTransactionBufferImpl.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestStorageContainerManagerRatisRoles.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java
copy
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/{ContainerBalancerStartSubcommand.java
=> ContainerBalancerConfigOptions.java} (54%)
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerEstimateSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
create mode 100644
hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestContainerBalancerAssessmentSubcommand.java
rename
hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/{datanode
=> }/TestContainerBalancerSubCommand.java (78%)
create mode 100644
hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestSafeModeCheckSubcommand.java
create mode 100644
hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java
create mode 100644
hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/CountSnapshotHandler.java
create mode 100644
hadoop-ozone/cli-shell/src/test/java/org/apache/hadoop/ozone/shell/snapshot/TestCountSnapshotHandler.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestECBlockChecksumComputer.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestOzoneCryptoInputStream.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/S3GatewayHealthCheckConfig.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/AssumeRoleResponseInfo.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/AwsRoleArnValidator.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/CallerIdentityInfo.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/IamSessionPolicyResolver.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/iam/package-info.java
(80%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/{om/helpers/OmMultipartCommitUploadPartInfo.java
=> snapshot/SnapshotBucketCount.java} (50%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/{om/helpers/OmMultipartCommitUploadPartInfo.java
=> snapshot/SnapshotCountResponse.java} (53%)
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestAssumeRoleResponseInfo.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestAwsRoleArnValidator.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestCallerIdentityInfo.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOMRatisHelper.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3STSUtils.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3STSUtilsCallerIdentity.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/TestIamSessionPolicyResolver.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/acl/iam/package-info.java
(80%)
create mode 100755 hadoop-ozone/dev-support/checks/errorprone.sh
create mode 100755
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-setup.sh
create mode 100755
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-smoketest.sh
create mode 100644
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris.yaml
create mode 100644
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/ranger-testlib.sh
copy hadoop-ozone/dist/src/main/compose/ozonesecure-ha/{.env => test-sts.sh}
(60%)
mode change 100644 => 100755
create mode 100644
hadoop-ozone/dist/src/main/smoketest/repair/om-update-transaction.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/health.robot
copy hadoop-ozone/dist/src/main/{compose/ozonesecure-ha/.env =>
smoketest/security/generate_oversized_session_policy.py} (60%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/security/mutate_sts_session_token.py
create mode 100644
hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts-multitenant.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts-polaris.sql
create mode 100644
hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.resource
create mode 100644
hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot
create mode 100644
hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsReadWriteValidator.java
create mode 100644
hadoop-ozone/httpfsgateway/src/test/java/org/apache/ozone/fs/http/server/TestHttpFSExceptionProvider.java
create mode 100644
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHealthCheck.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/package-info.java
(80%)
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/package-info.java
(80%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOfsGetFileStatusBenchmark.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestOmSCMNodesReconfiguration.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopFsReadWriteValidator.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestOmMixedWorkloadUnderDeletionBench.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotDefragSpaceSavings.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/KerberosTests.java
rename hadoop-ozone/{integration-test/src/test =>
mini-cluster/src/main}/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
(67%)
delete mode 100644
hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockDetailsUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMThreadContext.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3AssumeRoleRequest.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/{lifecycle/OMLifecycleSaveScanStateRequest.java
=> s3/security/S3DeleteRevokedSTSTokensRequest.java} (53%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3RevokeSTSTokenRequest.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{DummyOMClientResponse.java
=> s3/security/S3AssumeRoleResponse.java} (67%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{lifecycle/OMLifecycleSaveScanStateResponse.java
=> s3/security/S3DeleteRevokedSTSTokensResponse.java} (59%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{lifecycle/OMLifecycleSaveScanStateResponse.java
=> s3/security/S3RevokeSTSTokenResponse.java} (59%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RevokedSTSTokenCleanupService.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/EntryValue.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/FullDiffSequentialReader.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/SnapDiffJobStore.java
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/{
=> diff}/SnapshotDiffValueParser.java (92%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/package-info.java
(80%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenEncryption.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenSecretManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestEDEKCacheLoader.java
copy
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/{TestGrpcOzoneManagerServer.java
=> TestOMMetricsSetters.java} (51%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMPerformanceMetrics.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLeakDetection.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerS3Auth.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerSnapshotCount.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystemLegacyBucket.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/execution/TestOMExecutionFlow.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOMLockDetailsUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOMThreadContext.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferTransactionInfo.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/package-info.java
(80%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeApplyPerf.java
rename
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/{TestOMLifecycleConfigurationRequest.java
=> OMLifecycleConfigurationRequestTestBase.java} (99%)
copy
hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureProcessor.java
=>
ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/S3SecurityTestUtils.java}
(55%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3AssumeRoleRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3RevokeSTSTokenRequest.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/package-info.java
(80%)
delete mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
copy
hadoop-ozone/{common/src/test/java/org/apache/hadoop/ozone/om/exceptions/TestResultCodes.java
=>
ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMClientResponse.java}
(50%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestS3DerivedKey.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/security/TestS3AssumeRoleResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestRevokedSTSTokenCleanupService.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/package-info.java
(80%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/TestFullDiffSequentialReader.java
rename
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/{
=> diff}/TestSnapshotDiffValueParser.java (99%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestS3SecurityUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenEncryption.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenIdentifier.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenSecretManager.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/package-info.java
(80%)
copy
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/{O3fsDtFetcher.java
=> AbstractOzoneDtFetcher.java} (62%)
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3DtFetcher.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OfsDtFetcher.java
(71%)
create mode 100644
hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDtFetcher.java
copy
hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/MalformedResourceException.java
=>
ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestDtFetcherProviders.java}
(56%)
copy
hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/DummyOMClientResponse.java
=>
recon/src/test/java/org/apache/hadoop/ozone/recon/api/types/TestFeatureProvider.java}
(53%)
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconTaskStatusTableUpgradeAction.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/ReadConsistencyFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3ContentTypeFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayReadinessProbe.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{exception/OS3ExceptionMapper.java
=> S3STSHeadersResponseFilter.java} (51%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3STSHttpServer.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/WeldCdiIntegration.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java
copy
hadoop-ozone/{common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java
=>
s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyResult.java}
(68%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSException.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/{OS3ExceptionMapper.java
=> OSTSExceptionMapper.java} (82%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OSTSNotFoundExceptionMapper.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/ChunksValidator.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ReadConsistencyContext.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java
copy
hadoop-ozone/{cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Shell.java
=> s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/Application.java}
(53%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3AssumeRoleResponseXml.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3GetCallerIdentityResponseXml.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSConfigKeys.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEnabled.java
(66%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEnabledEndpointRequestFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/{s3secret/S3SecretEndpointBase.java
=> s3sts/S3STSEndpointBase.java} (61%)
copy
hadoop-ozone/{common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java
=>
s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSResponseMetadata.java}
(55%)
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/package-info.java
(62%)
copy hadoop-ozone/s3gateway/src/main/resources/webapps/{s3gateway =>
s3g-sts}/WEB-INF/web.xml (73%)
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestReadConsistencyFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3ContentTypeFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHttpServers.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayLivenessServlet.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketNotImplemented.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyActionsAudit.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCopyObjectAndUploadPartCopyLastModified.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListMultipartUploads.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3ActionOverrideForOwnerVerification.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3RequestContext.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSExceptions.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOSTSNotFoundExceptionMapper.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/SignatureTestUtils.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestChunksValidator.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/package-info.java
(80%)
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3secret/TestS3SecretEndpointBase.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/TestS3STSEndpoint.java
copy
hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
=>
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/package-info.java
(80%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
