This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a change to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
discard 4c7151b5e8 HDDS-6955. [Ozone-streaming] Add explicit stream flag in
ozone shell (#3559)
omit 2227d339d0 HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use
streaming to put EC key. (#3516)
omit 8315f08791 HDDS-6842. [Ozone-Streaming] Reduce the number of watch
requests in StreamCommitWatcher. (#3492)
omit 9d0b07e8c2 HDDS-6592. [Ozone-Streaming] Fix
ContainerStateMachine#applyTransaction assert error (#3315)
omit 51b5395786 HDDS-5666. Add option to createKey via streaming api in
Freon (#2574)
omit d72a8a09d5 HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at
the end of the stream. (#3229)
omit 88c4d59d9b HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput.
(#3195)
omit 181a17eab3 HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters.
(#3207)
omit a4913f9e0d HDDS-6461. Update Ratis version to 2.3.0-da5d868-SNAPSHOT.
(#3205)
omit ab61b71d35 HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
omit e63539309f HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
omit 3cc2a7e21b HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
omit 8f8f166615 HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
omit 366a0d2cef HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
omit 113a56e4ce HDDS-6298. Add XceiverServerRatis stream config (#3070)
omit 30f8311b0e HDDS-6138.[Ozone-Streaming] Define a limit on the size of
the retry bufferList. (#2946)
omit 84728e0cd0 HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT
version (#3059)
omit 72e1e02db8 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
omit 0db32a0223 HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
omit 2aacc7a889 HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
omit 1f56a45e75 HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
omit 7a6ea80e59 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
omit 2020c2266e HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
omit be9641f9fa HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
omit ddc8a2c3d6 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
omit 9204b35f2b HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
omit e186bfc268 HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
omit 47ae948f8f HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
omit 3daf3d7a8f HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
omit aea1d841fb HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
omit 2e85eeb0cd HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
omit bb4511840a HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
omit 3ea9af3d59 HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
omit b1f1051538 HDDS-5599. [Ozone-Streaming]drop BufferPool and
ChunkBuffer to avoid buffer copying (#2557)
omit d33d9cb021 HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
omit 52306308c7 HDDS-5480. [Ozone-Streaming] Client and server should
support stream setup. (#2452)
omit bda70049ff HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
omit b6e2143981 HDDS-5452. Add link method to ContainerStateMachine for
Ratis streaming (#2422)
omit 16bafc62d6 HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
add fac23c9f3a HDDS-6960. EC: Implement the Over-replication Handler
(#3572)
add 243efe25d3 HDDS-6938. handle NPE when removing prefixAcl (#3568)
add a02c6df497 HDDS-6900. Propagate TimeoutException for all SCM HA Ratis
calls. (#3564)
add ab923a3538 HDDS-6794. EC: Analyze and add putBlock even on non writing
node in the case of partial single stripe. (#3514)
add c48a7b09ae HDDS-6968. Addendum: [Multi-Tenant] Fix USER_MISMATCH error
even on correct user. (#3578)
add d088d199bf HDDS-6982. EC: Attempt to cleanup the RECOVERING container
when reconstruction failed at coordinator. (#3583)
add 7471d38f0a HDDS-6978. EC: Cleanup RECOVERING container on DN restarts
(#3585)
add 5d87d0eb75 HDDS-6901. Configure HDDS volume reserved as percentage of
the volume space. (#3532)
add 901a88debf HDDS-6991. Throw java.lang.NullPointerException when
running setOwner (#3586)
add 77cd1691a5 HDDS-6725. Close Rocks objects properly in
StorageContainerManager (#3549)
add f21e2a1e0f HDDS-6954. Migrate simple tests in ozone-recon to JUnit5
(#3556)
add e822caebe9 HDDS-6993. [Multi-Tenant] Set QuotaInBytes and
QuotaInNamespace during Tenant Create. (#3590)
add a3db2dfc50 HDDS-6992. ListKeys : SubPaths with a mixture of files and
dirs breaks sorted order (#3589)
add 2e8f5f30ea HDDS-6761. [SCM HA finalization] Handle restarts, crashes,
and leader changes. (#3534)
add 27745b54dd HDDS-6889. EC: put key command with EC replication can use
ReplicationConfig validator (#3565)
add aa3bbf9932 HDDS-6981. Add metrics for BlockDeletingService on Datanode
(#3580)
add e1b6b76f0a HDDS-6989: EC: Add Test for RECOVERING container cleanup
when failure. (#3587)
add 989ef1a9cf HDDS-6999. Correct volume parameter description for freon
test tools (#3592)
add b7066604d3 HDDS-6990. AuthorizerLockImpl.java#tryWriteLockInOMRequest:
move sanity args check to after tryWriteLockThrowOnTimeout(). (#3588)
add 6afe31a0c5 HDDS-6940. EC: Skip the EC container for balancer (#3547)
add 3f3577a452 HDDS-6909. [Multi-Tenant] Use RangerClient for Ranger
operations (#3576)
add 843fac2fb6 HDDS-6433. Refactor OMFailoverProxyProvider to provide a
base for OM and GrpcOM FailoverProxyProviders (#3389)
add e1f39bd506 HDDS-6998. Ignore keys with higher updateID in
OpenKeyDeleteRequest (#3596)
add 912ff42bd1 HDDS-6994. Update Contributing guide (#3593)
add 59d8a96267 Add missing recon address to docker-config file. (#3597)
add af0c473586 HDDS-7005. Recursive delete through bucket link deletes
linked data. (#3598)
add a66e43e283 HDDS-6987. EC: Implement RECOVERING Container Scrubber
(#3591)
add 49f2f3b526 HDDS-7006. Incorrect usedNamespace caused by exceeding
bytes quota (#3600)
add a09492cf8e HDDS-7012. Bump aws-java-sdk-s3 to 1.12.261 (#3602)
add 3a12bf811c HDDS-7007. Missing spaces in error messages (#3601)
add 9ebb4aa534 HDDS-6895. EC: ReplicationManager - Logic to process the
under replicated queue and assign work to DNs (#3599)
add 2395d04fcb HDDS-6904. Cleanup customer related events in S3G logs
(#3531)
add 0a019a4d95 HDDS-7020. EC: ReplicationManager - skip processing open
containers (#3608)
add 5e17998eef HDDS-7009. Avoid calling getBucket in object APIs (#3594)
add 1e546103f0 HDDS-7016. EC: Implement the Over replication Processor
(#3610)
add 0886f62e6c HDDS-6976. 0GB data moved by container balancer after
successful iteration (#3604)
add 6ee6c35767 HDDS-6807. Remove tcnative.version property (#3605)
add 2795fda0c5 HDDS-6995. Update ranger-intg to v2.3.0 (#3603)
add 3c6a128ca2 HDDS-7015. Ensure excluded nodes have network location when
used in placement policies (#3606)
add c57c9bca25 HDDS-7032. RDBStore#getUpdatesSince should throw
SequenceNumberNotFou… (#3613)
add f5ab00c6bf HDDS-5251. Refactor ContainerTestHelper factory methods
(#3612)
add c55c2dc474 HDDS-7024. Add queue size metric for OM double buffer
(#3611)
add ca26bf00df HDDS-7034. File handle leak on datanode in PutBlock (#3616)
add c18295411d HDDS-6871. Bump netty to 4.1.79 (#3505)
add 8f16515fcc HDDS-7036. Enable RocksDB stats in monitoring sample config
(#3619)
add 7de7da5056 HDDS-7021. [Multi-Tenant] Fix warning message when OM BG
Sync is unable to get Ozone service policy version from Ranger (#3609)
add 58c3166094 HDDS-6767. Add network location in Recon's datanode page
(#3429)
add 11cdb25fec HDDS-7041. Avoid ConcurrentModificationException in
RepeatedOmKeyInfo (#3617)
add bea83f206f HDDS-7046. Leader count missing in Recon (#3621)
add d1cdf9994e HDDS-7042. Rebuilding tenant cache omits empty tenants
(#3618)
add 72ca001178 HDDS-6980. support block composite-crc checksum (#3581)
add af059010b6 HDDS-6956. Initialize ozoneLockProvider earlier in
OzoneManager (#3555)
add d80b6293e5 HDDS-6728. SCM UI not showing correct HA roles (#3401)
add 97c6832de9 HDDS-7057. EC: ReplicationManager - Over replication
handler should set repIndex on delete cmds (#3628)
add 9f188e38b2 HDDS-7056. EC: Ensure replica index is maintained when
replicating a container (#3627)
add c913760611 HDDS-7052. Topology Dynamic List document issue (#3624)
add 4393525367 HDDS-7055. NPE in ec.reconstruction.TokenHelper (#3630)
add 6b5dc0fa68 HDDS-6899. EC: remove warnings and errors from console
during online reconstruction of data. (#3632)
add 0485fda4f7 HDDS-7050. Recon "/missing" endpoint return limit too high
(#3622)
add 9e8c1900ba HDDS-7059. S3G should not call lookup twice for Get Object
(#3633)
add daf2578256 HDDS-6533. support balancing EC container (#3455)
add 57d620dbd4 HDDS-7068: EC: Prematurely re-throwed the exception in
reconstruction cleanup loop. (#3639)
add 511977ad67 HDDS-7048. EC: Add debug logging with exception info when
stripe write failed (#3625)
add 2f03b16f87 HDDS-7069. EC: ReplicationManager - Track nodes already
used when handing under replication (#3640)
add 334c6ee65b HDDS-7073. Incomplete audit log for KeyBlockToDelete (#3642)
add 3b89d4daac HDDS-5564. feature/PrefixFSO.md translation (#2582)
add ccf1b7c18f HDDS-6951. Replace bucket.listKeys() with
bucket.listStatus() in OmBucketReadWriteKeyOps (#3554)
add 2396611e3c HDDS-7044. Ignore pr_title_check for selective checks
(#3620)
add fa61f9e663 HDDS-7077. EC: Fix block deletion not allowed due to
missing pipelineID (#3644)
add 15c9a5bca3 HDDS-7081. EC: ReplicationManager - UnderRep handler should
handle duplicate indexes (#3649)
add 348fcb6157 HDDS-7051. Fix offset Condition in ECKeyOutputStream (#3623)
add ef218d8b68 HDDS-6662. Provide basic hash code generator for
OBSKeyPathLockStrategy (#3358)
add 4459f47c11 HDDS-7063. Increment createContainer metric when container
is created while writing chunk (#3643)
add 8f23e6bd54 HDDS-4970. Significant overhead when DataNode is
over-subscribed (#3654)
add 51897d4aa1 HDDS-7023. Add CLI for reseting the expired deleted block
txn (#3615)
add 192eff11a9 HDDS-7087. Manage RocksObjects to detect leaks (#3656)
add 8e11723984 HDDS-7085. Update gRPC to 1.48.1 to address OOM bug (#3659)
add 7d50d29d51 HDDS-7066. Include number of tests requested in Freon
output (#3638)
add 8240341efd HDDS-6997. Add support for "echo" lookup to measure raw
performance in OM (#3614)
add 1afaa652c1 HDDS-7047. Add Recon to ozone-topology sample compose
environment (#3641)
add b696473437 HDDS-7109. SCM should close LayoutVersionManager in
stopping stage (#3666)
add fded237013 HDDS-7053. Add client-side pipelines distribution metrics
(#3626)
add dd699d952d HDDS-7043. Disallow ellipsis in PR title (#3651)
add 9583e47331 HDDS-6581. Introduce KEY_PATH_LOCK in OMKeyCreateRequest
class (#3560)
add 64032cc223 HDDS-7022. EC: Open EC container are not closed when SCM
container was already closed. (#3668)
add 658e139b45 HDDS-7116. Avoid leaking RocksObject from DBProfile (#3673)
add 13ccc5f8d3 HDDS-5504. Refactor namespace summaries (du, dist & counts)
for upcoming legacy FS buckets support (#3309)
add 5adbef2135 HDDS-6127. file checksum to support both CRC32 and CRC32C.
(#3570)
add 5ba6728f7c HDDS-6726. Close RocksObject in Recon and tools (#3678)
add f9bd49e043 HDDS-7102. Exclude unnecessary RangerClient dependencies
(#3669)
add 2554228edc HDDS-7033. Include committed space in log for disk out of
space (#3636)
add ec3d55f6b5 HDDS-7127. Fix Transparent Data Encryption documentation
(#3685)
add 5f84fd1c9f HDDS-7074. DN EndpointStateMachineMBean to add getType()
(#3650)
add 636d23615c HDDS-7120. Prometheus displays only one volume_io_stats per
datanode (#3682)
add ab608c5d46 HDDS-7045. Election info is out of date in Recon (#3677)
add 82b6441c6f HDDS-7076. Log container file path when container cannot be
written. (#3684)
add be2fe71d69 HDDS-6458. Add ReplicationConfig options for bucketArgs in
MiniOzoneChaosCluster (#3686)
add 5d8573a970 HDDS-7134. NPE when Ranger client throws
RangerServiceException without Status. (#3689)
add 3efb039091 HDDS-7113. Support overriding response header values (#3680)
add 749b526503 HDDS-7147. DirectoryDeletingService interval unit mismatch
(#3695)
add 01afc1ff33 HDDS-7136. Memory leak due to ChunkInputStream.close() not
releasing buffer (#3692)
add 20863d3148 HDDS-7139. INVALID_TOKEN is handled as internal error in
S3G (#3698)
add e9689764d3 HDDS-7108. Include supported EC data-parity combinations in
error message (#3665)
add 4fd698f05e Multiple RPC clients per test run. (#3696)
add 38c767fa5f HDDS-7135. ofs file input stream should support
StreamCapabilities interface (#3694)
add b11c5f12f8 HDDS-5779. Create s3g bucket creation performance test
(#3629)
add b2b19de090 HDDS-7154. Fixed code issues in
org.apache.hadoop.hdds.client reported by sonar (#3700)
add 9fc0f9801d HDDS-7150. Freon fail-at-end option caused not shutdown
(#3697)
add a1ba9eaa55 HDDS-7108. Make ReplicationConfig validation error message
type-specific (#3702)
add ac45a349ef HDDS-7159. Upgrade ubuntu version from 18.04 to 20.04
(#3707)
add 5ea8e6ab3f HDDS-7157. Log the error msg explicitly in
BlockDeletingService (#3704)
add df9ed54ba1 HDDS-7159. Use Java 8 for integration tests (#3708)
add 8bb3d6e7af HDDS-7151. Avoid using GeneratedMessage in non-generated
code (#3699)
add 132bec2363 HDDS-7153. User defined override of ozone shell hardcoded
values for log files (#3706)
add 59a795b2cc HDDS-7164. Skip installing awscli if already present (#3712)
add a0b996651d HDDS-6456. Update RocksDB Version to 7.x (#3690)
add a828dba870 HDDS-7104. Remove WARNINGs while running ozone fs/CLI
commands. (#3671)
add 5c18a5624f HDDS-7162. Add Double in ConfigType (#3710)
add 3cf882e93c HDDS-6243. sourceToTargetMap in ContainerBalancer doesn't
support multiple entries with same source. (#3485)
add 20c45aee39 HDDS-7161. Make Checksum.int2ByteString() zero-copy. (#3709)
add 9333f1e5bf HDDS-7166. Memory leak in Recon when replacing DB for new
checkpoint (#3713)
add e1f4548d35 HDDS-7173. Fix
TestReconOmMetadataManagerImpl#testUpdateOmDB (#3719)
add 1caf752269 HDDS-7172. Intermittent failure in Close pipeline smoketest
(#3717)
add 93ae1cfcd5 HDDS-7132. GetFileStatus returns NULL for some paths.
(#3687)
add f8ca8c5aff HDDS-6967. EC: Handle maintenance replicas in
ECUnderReplicationHandler (#3661)
add 4f3e767b69 HDDS-6493. Improve TestOmKeyInfoCodec error handling (#3718)
add 2f757f6287 HDDS-7165. Remove AbstractContractUnbufferTest (#3714)
add 4585bbe89d HDDS-2954. Support admin groups (#3693)
add 48230b6fe0 HDDS-7158. ldb cli command supports to scan container V3.
(#3705)
add 3dcd68ca3d HDDS-6621. Include annotation processor module in regular
project hierarchy (#3326)
add 09e89fd67e HDDS-6312. Use KeyPrefixContainer table to accelerate the
process of DELETE/UPDATE events (#3082)
add 537bac8887 HDDS-7176. Sort the container replica according to the
replica index (#3720)
add b7c13dba3c HDDS-7163. Ozone debug container CLI supports container
export. (#3715)
add 3e0cb27632 HDDS-6975. EC: Define the value of Maintenance Redundancy
for EC containers (#3723)
add 919e7755ab HDDS-7186. Add support for Rocksdb user info LOG configure.
(#3731)
add b093984f16 HDDS-7175. Timed out tests missing from bundle (#3721)
add abda1f2786 HDDS-7125. Inaccurate numBlocksDeletion when
getTransactions (#3681)
add 492ae919a2 HDDS-7171. Add SupportedSourceVersion in annotation
processors (#3724)
add a8efd9e6a7 HDDS-7200. Fixed inaccurate get request status code (#3733)
add 3d3e9eaf00 HDDS-7197. Schema V3 RocksDB instance statitics register as
metrics collision (#3734)
add ec9327de59 HDDS-7201.
testContainerIsReplicatedWhenAllNodesGotoMaintenance is failing frequently
(#3737)
add 2ce64319a4 HDDS-7143. Recon: Fix left to right scrolling on Pipelines
page (#3735)
add ab4288ccb8 HDDS-7191. Separate prop for s3 admin (#3732)
add 25dc267115 HDDS-7086. DataNode UI to show failed volumes too (#3740)
add e84aa4c4ea HDDS-2476. Share more code between metadata and data
scanners. (#3727)
add 8f32c7e05a HDDS-7174. Migrate SCM integration tests to JUnit 5 (#3722)
add 4030265b0c HDDS-7204. EC: Schedule UnderReplicatedProcessor and
OverReplicatedProcessor threads in RM instead of StorageContainerManager (#3738)
new b3495e8d39 HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
new 425bc95539 HDDS-5452. Add link method to ContainerStateMachine for
Ratis streaming (#2422)
new 444b7b3840 HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
new 0a4bbf5c7b HDDS-5480. [Ozone-Streaming] Client and server should
support stream setup. (#2452)
new 18db09d7d4 HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
new 6b51811d1f HDDS-5599. [Ozone-Streaming]drop BufferPool and
ChunkBuffer to avoid buffer copying (#2557)
new 74c5f0f5fc HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
new b7b16c6977 HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
new 10b14300d2 HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
new 79507fe72f HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
new 508fda8744 HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
new ca7a926165 HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
new 311b366526 HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
new bb4eef2f7c HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
new 65ec5817e2 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
new 9a16e41e1f HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
new 4194bd6ce0 HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
new 3b9141e18d HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
new 750c5f475a HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
new 1ecb774a39 HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
new 6bd6cc7f2d HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
new 151ee53cd7 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
new fdd7b64156 HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT
version (#3059)
new 54361e5a4d HDDS-6138.[Ozone-Streaming] Define a limit on the size of
the retry bufferList. (#2946)
new f59d30da45 HDDS-6298. Add XceiverServerRatis stream config (#3070)
new 576c359114 HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
new 9d24f58704 HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
new de690f6c25 HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
new c1bc998af6 HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
new caa3b30a1e HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
new 22e77a7b3d HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters.
(#3207)
new 0a432aa0eb HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput.
(#3195)
new 1dddbec252 HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at
the end of the stream. (#3229)
new a2501c3de3 HDDS-5666. Add option to createKey via streaming api in
Freon (#2574)
new f850883767 HDDS-6592. [Ozone-Streaming] Fix
ContainerStateMachine#applyTransaction assert error (#3315)
new 4c064f42aa HDDS-6842. [Ozone-Streaming] Reduce the number of watch
requests in StreamCommitWatcher. (#3492)
new 04ebe89843 HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use
streaming to put EC key. (#3516)
new c84da64fbc HDDS-6955. [Ozone-streaming] Add explicit stream flag in
ozone shell (#3559)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4c7151b5e8)
\
N -- N -- N refs/heads/HDDS-4454 (c84da64fbc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 38 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/close-pending.yaml | 2 +-
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 26 +-
CONTRIBUTING.md | 146 +--
dev-support/annotations/pom.xml | 114 --
.../services/javax.annotation.processing.Processor | 16 -
dev-support/ci/pr_title_check.bats | 8 +
dev-support/ci/pr_title_check.sh | 2 +
dev-support/ci/selective_ci_checks.bats | 12 +
dev-support/ci/selective_ci_checks.sh | 10 +-
hadoop-hdds/annotations/pom.xml | 34 +
.../annotations/ReplicateAnnotationProcessor.java | 80 ++
.../RequestFeatureValidatorProcessor.java | 0
.../org/apache/ozone/annotations/package-info.java | 0
.../hadoop/hdds/scm/ContainerClientMetrics.java | 138 +++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 10 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 13 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 42 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 1 +
.../hdds/scm/storage/ECBlockOutputStream.java | 7 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 7 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 4 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hdds/scm/TestContainerClientMetrics.java | 116 ++
.../storage/TestBlockOutputStreamCorrectness.java | 4 +-
hadoop-hdds/common/pom.xml | 4 +
.../hdds/client/DefaultReplicationConfig.java | 2 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 2 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 9 -
.../org/apache/hadoop/hdds/client/QuotaList.java | 19 +-
.../hadoop/hdds/client/ReplicationConfig.java | 4 +-
.../hdds/client/ReplicationConfigValidator.java | 34 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 5 +-
.../hdds/scm/PlacementPolicyValidateProxy.java | 51 +
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 3 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +
.../protocol/StorageContainerLocationProtocol.java | 9 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 32 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 6 +
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 3 +-
.../upgrade/AbstractLayoutVersionManager.java | 17 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 59 +-
.../DefaultUpgradeFinalizationExecutor.java | 6 +-
.../services/javax.annotation.processing.Processor | 2 +
.../common/src/main/resources/ozone-default.xml | 55 +
.../hdds/client/TestECReplicationConfig.java | 29 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 36 +-
.../client/TestReplicationConfigValidator.java | 14 +-
.../hadoop/hdds/conf/SimpleConfiguration.java | 13 +
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 10 +
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 17 +
.../ozone/container/ContainerTestHelper.java | 65 +-
.../InjectedUpgradeFinalizationExecutor.java | 3 -
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 105 +-
.../hadoop/ozone/upgrade/UpgradeTestUtils.java | 59 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +
.../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 9 +
.../hadoop/hdds/conf/ConfigurationSource.java | 8 +
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
.../hadoop/hdds/conf/ConfigurationExample.java | 13 +
.../hdds/conf/TestConfigurationReflectionUtil.java | 4 +
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../datanode/metadata/DatanodeCRLStoreImpl.java | 13 +-
.../helpers/BlockDeletingServiceMetrics.java | 102 ++
.../ozone/container/common/impl/ContainerSet.java | 80 +-
.../container/common/impl/HddsDispatcher.java | 25 +-
.../container/common/report/ReportManager.java | 4 +-
.../container/common/report/ReportPublisher.java | 6 +-
.../common/report/ReportPublisherFactory.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 124 ++
.../common/statemachine/EndpointStateMachine.java | 16 +-
.../statemachine/EndpointStateMachineMBean.java | 2 +
.../common/statemachine/StateContext.java | 56 +-
.../states/endpoint/HeartbeatEndpointTask.java | 6 +-
.../container/common/utils/HddsVolumeUtil.java | 8 +-
.../common/utils/db/DatanodeDBProfile.java | 43 +-
.../common/volume/AvailableSpaceFilter.java | 84 ++
.../volume/CapacityVolumeChoosingPolicy.java | 48 +-
.../ozone/container/common/volume/HddsVolume.java | 14 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 28 +-
.../common/volume/VolumeChoosingUtil.java | 49 +
.../container/common/volume/VolumeIOStats.java | 10 +-
.../ozone/container/common/volume/VolumeInfo.java | 33 +-
.../container/common/volume/VolumeInfoMetrics.java | 5 +
.../ozone/container/common/volume/VolumeUsage.java | 9 +-
.../reconstruction/ECContainerOperationClient.java | 40 +
.../ECReconstructionCoordinator.java | 69 +-
.../ECReconstructionCoordinatorTask.java | 6 +
.../container/ec/reconstruction/TokenHelper.java | 6 +-
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 1 +
.../ozone/container/keyvalue/KeyValueHandler.java | 24 +-
.../container/keyvalue/helpers/ChunkUtils.java | 50 +-
.../keyvalue/impl/ChunkManagerFactory.java | 12 +-
.../background/BlockDeletingService.java | 60 +-
.../StaleRecoveringContainerScrubbingService.java | 93 ++
.../container/metadata/AbstractDatanodeStore.java | 52 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 9 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 7 +-
.../ozoneimpl/AbstractContainerScanner.java | 136 +++
...s.java => AbstractContainerScannerMetrics.java} | 42 +-
.../container/ozoneimpl/ContainerDataScanner.java | 135 +--
.../ozoneimpl/ContainerDataScannerMetrics.java | 69 ++
.../ozoneimpl/ContainerDataScrubberMetrics.java | 120 --
.../ozoneimpl/ContainerMetadataScanner.java | 112 +-
.../ozoneimpl/ContainerMetadataScannerMetrics.java | 44 +
.../ozone/container/ozoneimpl/ContainerReader.java | 15 +-
...ion.java => ContainerScannerConfiguration.java} | 12 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 92 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 3 +-
.../protocol/StorageContainerDatanodeProtocol.java | 3 +-
.../protocol/commands/CommandForDatanode.java | 4 +-
.../protocol/commands/DeleteContainerCommand.java | 20 +-
.../commands/ReconstructECContainersCommand.java | 15 +-
.../commands/ReplicateContainerCommand.java | 20 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 4 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +-
.../webapps/hddsDatanode/dn-overview.html | 2 +
.../hadoop/ozone/TestHddsDatanodeService.java | 11 +
.../container/common/TestBlockDeletingService.java | 52 +-
.../common/TestKeyValueContainerData.java | 7 +
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 181 +++
.../impl/TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../common/report/TestReportPublisher.java | 6 +-
.../common/statemachine/TestStateContext.java | 22 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 8 +-
.../container/common/utils/TestHddsVolumeUtil.java | 8 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 21 +-
.../common/volume/TestReservedVolumeSpace.java | 149 +++
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 33 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 92 ++
.../container/common/volume/TestVolumeSet.java | 4 +-
.../container/keyvalue/TestKeyValueContainer.java | 36 -
.../keyvalue/TestKeyValueContainerCheck.java | 15 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 22 +-
.../container/ozoneimpl/TestContainerReader.java | 26 +-
...java => TestContainerScannerConfiguration.java} | 28 +-
...trics.java => TestContainerScannerMetrics.java} | 20 +-
.../ReplicationSupervisorScheduling.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../TestReconstructionECContainersCommands.java | 16 +-
hadoop-hdds/docs/content/concept/Recon.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 69 ++
.../docs/content/feature/S3-Multi-Tenancy-Setup.md | 27 +-
hadoop-hdds/docs/content/feature/Topology.md | 4 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 5 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 12 +-
.../docs/content/security/SecuringTDE.zh.md | 6 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 13 +
.../certificate/authority/CertificateStore.java | 6 +-
.../certificate/authority/DefaultCAServer.java | 8 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 87 ++
.../hadoop/hdds/server/http/BaseHttpServer.java | 8 +-
.../hadoop/hdds/utils/DBCheckpointServlet.java | 11 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 30 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 22 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 8 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 61 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 125 +-
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 12 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 8 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 10 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 28 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 72 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 30 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 176 +--
.../apache/hadoop/hdds/utils/db/TableConfig.java | 27 +-
.../db/managed/ManagedBlockBasedTableConfig.java | 74 ++
.../hdds/utils/db/managed/ManagedBloomFilter.java | 20 +-
.../hdds/utils/db/managed/ManagedCheckpoint.java | 33 +
.../db/managed/ManagedColumnFamilyOptions.java | 83 ++
.../hdds/utils/db/managed/ManagedDBOptions.java | 20 +-
.../hdds/utils/db/managed/ManagedEnvOptions.java | 19 +-
.../hdds/utils/db/managed/ManagedFlushOptions.java | 20 +-
.../managed/ManagedIngestExternalFileOptions.java | 21 +-
.../hdds/utils/db/managed/ManagedLRUCache.java | 23 +-
.../hdds/utils/db/managed/ManagedObject.java | 35 +-
.../hdds/utils/db/managed/ManagedOptions.java | 20 +-
.../hdds/utils/db/managed/ManagedReadOptions.java | 23 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 81 ++
.../utils/db/managed/ManagedRocksIterator.java | 34 +
.../db/managed/ManagedRocksObjectMetrics.java | 60 +
.../utils/db/managed/ManagedRocksObjectUtils.java | 52 +
.../hadoop/hdds/utils/db/managed/ManagedSlice.java | 29 +-
.../utils/db/managed/ManagedSstFileWriter.java | 26 +-
.../hdds/utils/db/managed/ManagedStatistics.java | 19 +-
.../db/managed/ManagedTransactionLogIterator.java | 23 +-
.../hdds/utils/db/managed/ManagedWriteBatch.java | 28 +-
.../hdds/utils/db/managed/ManagedWriteOptions.java | 23 +-
.../hadoop/hdds/utils/db/managed/package-info.java | 35 +
.../security/token/TestBlockTokenVerifier.java | 3 +-
.../security/token/TestContainerTokenVerifier.java | 3 +-
.../token/TestOzoneBlockTokenSecretManager.java | 51 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 8 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 18 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 15 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 13 +-
.../src/main/proto/ScmAdminProtocol.proto | 12 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +
hadoop-hdds/pom.xml | 7 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 44 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 3 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 16 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 42 +-
.../scm/block/DeletedBlockLogStateManager.java | 11 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 39 +-
.../container/AbstractContainerReportHandler.java | 10 +-
.../scm/container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 8 +-
.../hdds/scm/container/ContainerManagerImpl.java | 11 +-
.../hdds/scm/container/ContainerReplica.java | 4 +-
.../hdds/scm/container/ContainerReportHandler.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 7 +-
.../scm/container/ContainerStateManagerImpl.java | 42 +-
.../IncrementalContainerReportHandler.java | 4 +-
.../scm/container/RatisContainerReplicaCount.java | 1 +
.../balancer/AbstractFindTargetGreedy.java | 14 +-
.../scm/container/balancer/ContainerBalancer.java | 221 ++--
.../ContainerBalancerSelectionCriteria.java | 71 +-
.../FindTargetGreedyByNetworkTopology.java | 7 +-
.../balancer/FindTargetGreedyByUsageInfo.java | 6 +-
.../algorithms/SCMContainerPlacementCapacity.java | 7 +-
.../algorithms/SCMContainerPlacementRackAware.java | 2 +-
.../SCMContainerPlacementRackScatter.java | 16 +-
.../algorithms/SCMContainerPlacementRandom.java | 6 +-
.../AbstractOverReplicationHandler.java | 110 ++
.../replication/ContainerHealthResult.java | 12 +
.../{ => replication}/ContainerReplicaCount.java | 5 +-
.../replication/ECContainerHealthCheck.java | 1 -
.../{ => replication}/ECContainerReplicaCount.java | 56 +-
.../replication/ECOverReplicationHandler.java | 170 +++
.../replication/ECUnderReplicationHandler.java | 160 ++-
.../replication/LegacyReplicationManager.java | 73 +-
.../replication/OverReplicatedProcessor.java | 147 +++
.../container/replication/ReplicationManager.java | 337 +++++-
.../replication/UnderReplicatedProcessor.java | 158 +++
...ndler.java => UnhealthyReplicationHandler.java} | 11 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 4 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 3 +
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 8 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 1 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 32 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 15 +-
.../hdds/scm/ha/StatefulServiceStateManager.java | 3 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 11 +-
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 2 +-
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 12 +-
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 3 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 3 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 3 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 7 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 3 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 29 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 50 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 13 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 14 +-
.../scm/pipeline/WritableContainerFactory.java | 3 +-
.../scm/pipeline/WritableContainerProvider.java | 4 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
.../pipeline/WritableRatisContainerProvider.java | 3 +-
...inerLocationProtocolServerSideTranslatorPB.java | 19 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 22 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 5 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 136 ++-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 1 +
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 4 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 10 +-
.../hdds/scm/server/StorageContainerManager.java | 36 +-
.../scm/server/upgrade/FinalizationCheckpoint.java | 25 +-
.../scm/server/upgrade/FinalizationManager.java | 16 +
.../server/upgrade/FinalizationManagerImpl.java | 38 +
.../server/upgrade/FinalizationStateManager.java | 15 +-
.../upgrade/FinalizationStateManagerImpl.java | 150 ++-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 88 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 11 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 18 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 198 +++-
.../container/TestCloseContainerEventHandler.java | 7 +-
.../scm/container/TestContainerManagerImpl.java | 7 +-
.../scm/container/TestContainerReportHandler.java | 25 +-
.../scm/container/TestContainerStateManager.java | 12 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../container/balancer/TestContainerBalancer.java | 251 ++--
.../container/balancer/TestFindTargetStrategy.java | 3 +-
.../TestSCMContainerPlacementCapacity.java | 10 +
.../TestSCMContainerPlacementRackAware.java | 5 +-
.../TestSCMContainerPlacementRackScatter.java | 6 +-
.../container/replication/ReplicationTestUtil.java | 120 +-
.../TestECContainerReplicaCount.java | 89 +-
.../replication/TestECOverReplicationHandler.java | 164 +++
.../replication/TestECUnderReplicationHandler.java | 306 +++++
.../replication/TestLegacyReplicationManager.java | 95 +-
.../replication/TestOverReplicatedProcessor.java | 129 ++
.../TestRatisContainerReplicaCount.java | 24 +-
.../replication/TestReplicationManager.java | 78 +-
.../replication/TestUnderReplicatedProcessor.java | 184 +++
.../states/TestECUnderReplicationHandler.java | 214 ----
.../hdds/scm/ha/TestBackgroundSCMService.java | 7 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 3 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 2 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 16 +-
.../TestPipelineDatanodesIntersection.java | 3 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 8 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 14 +-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 33 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 6 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 2 +-
.../pipeline/TestWritableECContainerProvider.java | 28 +-
.../hdds/scm/update/server/MockCRLStore.java | 4 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 6 +-
.../hdds/scm/upgrade/TestScmFinalization.java | 52 +-
.../org/apache/ozone/test/GenericTestUtils.java | 49 +-
.../hdds/scm/cli/ContainerOperationClient.java | 5 +
.../hdds/scm/cli/container/InfoSubcommand.java | 6 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 22 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 187 ++-
.../apache/hadoop/ozone/client/OzoneClient.java | 13 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +-
.../checksum/AbstractBlockChecksumComputer.java | 5 +-
.../client/checksum/BaseFileChecksumHelper.java | 40 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 82 +-
.../checksum/ReplicatedFileChecksumHelper.java | 20 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 19 +-
.../client/io/BlockOutputStreamEntryPool.java | 20 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 46 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 7 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 34 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 28 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 26 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 72 +-
.../TestReplicatedBlockChecksumComputer.java | 56 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 11 +-
hadoop-ozone/common/pom.xml | 6 -
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 6 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 102 +-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 240 ++++
...vider.java => OMFailoverProxyProviderBase.java} | 304 ++---
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 20 +
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 7 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 18 +-
.../hadoop/ozone/om/multitenant/AccessPolicy.java | 2 +-
.../hadoop/ozone/om/multitenant/OzoneTenant.java | 10 +-
.../ozone/om/multitenant/RangerAccessPolicy.java | 4 +-
.../apache/hadoop/ozone/om/multitenant/Tenant.java | 12 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 26 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 27 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 20 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 10 +-
.../OMInterServiceProtocolClientSideImpl.java | 12 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 38 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 12 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 347 ++++++
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 508 +++-----
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 42 +-
.../dev-support/checks/_mvn_unit_report.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
.../compose/ozone-topology/docker-compose.yaml | 15 +
.../src/main/compose/ozone-topology/docker-config | 5 +
.../dist/src/main/compose/ozone-topology/test.sh | 2 +
.../dist/src/main/compose/ozone/monitoring.conf | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 14 -
.../src/main/compose/ozonesecure/docker-config | 26 +-
.../ozonesecure/mockserverInitialization.json | 98 --
.../main/compose/upgrade/compose/ha/docker-config | 1 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 6 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 13 +-
.../src/main/smoketest/admincli/pipeline.robot | 5 +-
.../src/main/smoketest/freon/echoRPCLoad.robot | 42 +
.../dist/src/main/smoketest/recon/recon-api.robot | 14 +
.../main/smoketest/recon/recon-fso-nssummary.robot | 146 +++
.../dist/src/main/smoketest/s3/commonawslib.robot | 2 +
.../dist/src/main/smoketest/s3/freon.robot | 44 +
.../dist/src/main/smoketest/s3/objecthead.robot | 47 +
.../dist/src/main/smoketest/s3/objectputget.robot | 6 +
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../smoketest/security/ozone-secure-tenant.robot | 4 +
hadoop-ozone/dist/src/shell/ozone/ozone | 3 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 13 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 22 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 28 +-
.../network-tests/src/test/blockade/README.md | 4 +-
.../fs/contract/AbstractContractUnbufferTest.java | 160 ---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 83 ++
.../hdds/scm/TestSCMDbCheckpointServlet.java | 31 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 32 +-
.../TestContainerStateManagerIntegration.java | 2 +-
.../metrics/TestSCMContainerManagerMetrics.java | 83 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 42 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 43 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 22 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 5 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 37 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 35 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 151 ++-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 230 +---
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 272 +++++
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 352 ++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 37 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 28 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 11 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../hadoop/ozone/TestStorageContainerManager.java | 6 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 3 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 55 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 68 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 20 +
.../ozone/container/TestECContainerRecovery.java | 286 +++++
.../commandhandler/TestBlockDeletion.java | 101 +-
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 17 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 6 +-
.../TestDataScanner.java} | 31 +-
.../hadoop/ozone/om/OmFailoverProxyUtil.java | 4 +-
.../om/{TestOmLDBCli.java => TestLDBCli.java} | 124 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 302 +++--
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 7 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 18 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 36 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 6 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../ozone/om/multitenant/RangerUserRequest.java | 278 +++++
...estMultiTenantAccessAuthorizerRangerPlugin.java | 269 -----
.../om/multitenant/TestMultiTenantVolume.java | 29 +
.../om/multitenant/TestRangerBGSyncService.java | 349 +++---
.../hadoop/ozone/scm/TestAllocateContainer.java | 39 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 26 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 84 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 51 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 3 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 3 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 37 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 36 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 14 +
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 22 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 103 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 19 +-
.../scm/node/TestDecommissionAndMaintenance.java | 212 ++--
.../hadoop/ozone/scm/node/TestQueryNode.java | 29 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 35 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 28 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 72 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 27 +-
.../TestResetDeletedBlockRetryCountShell.java | 169 +++
.../src/main/proto/OmClientProtocol.proto | 30 +
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 45 +-
hadoop-ozone/ozone-manager/pom.xml | 22 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 6 +-
.../hadoop/ozone/om/OMMultiTenantManager.java | 112 +-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 350 +++---
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 61 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 173 ++-
.../om/hashcodegenerator/OMHashCodeGenerator.java | 20 +-
.../StringOMHashCodeGeneratorImpl.java | 16 +-
.../ozone/om/hashcodegenerator}/package-info.java | 8 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 41 +-
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 15 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 8 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 3 +-
.../ozone/om/multitenant/CachedTenantState.java | 41 +
.../InMemoryMultiTenantAccessController.java | 49 +-
.../multitenant/MultiTenantAccessAuthorizer.java | 282 -----
.../MultiTenantAccessAuthorizerDummyPlugin.java | 194 ---
.../MultiTenantAccessAuthorizerRangerPlugin.java | 854 -------------
.../multitenant/MultiTenantAccessController.java | 190 ++-
.../om/multitenant/OMRangerBGSyncService.java | 194 +--
.../RangerClientMultiTenantAccessController.java | 266 ++++-
.../RangerRestMultiTenantAccessController.java | 31 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 +-
.../metrics/OzoneManagerDoubleBufferMetrics.java | 13 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMEchoRPCRequest.java} | 28 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 7 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 7 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 11 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +
.../om/request/key/OMOpenKeysDeleteRequest.java | 8 +
.../request/key/acl/prefix/OMPrefixAclRequest.java | 6 +
.../request/s3/security/S3SecretRequestHelper.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 2 +
.../protocolPB/OzoneManagerRequestHandler.java | 44 +
.../ozone/security/acl/OzoneNativeAuthorizer.java | 22 +-
.../hadoop/ozone/om/TestOMMultiTenantManager.java | 32 +-
.../ozone/om/TestOMMultiTenantManagerImpl.java | 126 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 42 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 11 +-
.../TestMultiTenantAccessController.java | 97 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +
.../bucket/TestOMBucketSetPropertyRequest.java | 8 +-
.../om/request/key/TestOMKeyCommitRequest.java | 43 +
.../om/request/key/TestOMKeyCreateRequest.java | 47 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 5 +
.../request/key/TestOMOpenKeysDeleteRequest.java | 71 +-
.../om/request/key/TestOMPrefixAclRequest.java | 66 ++
.../s3/security/TestS3GetSecretRequest.java | 207 ++--
.../volume/TestOMVolumeSetQuotaRequest.java | 8 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../security/acl/TestOzoneAdministrators.java | 66 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 3 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 3 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 11 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 40 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 17 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 +
.../hadoop/ozone/recon/ReconControllerModule.java | 4 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 2 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 929 +--------------
.../hadoop/ozone/recon/api/NodeEndpoint.java | 1 +
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 57 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 12 +-
.../recon/api/handlers/BucketEntityHandler.java | 152 +++
.../ozone/recon/api/handlers/BucketHandler.java | 207 ++++
.../recon/api/handlers/DirectoryEntityHandler.java | 150 +++
.../ozone/recon/api/handlers/EntityHandler.java | 342 ++++++
.../ozone/recon/api/handlers/FSOBucketHandler.java | 279 +++++
.../ozone/recon/api/handlers/KeyEntityHandler.java | 93 ++
.../recon/api/handlers/RootEntityHandler.java | 176 +++
.../recon/api/handlers/UnknownEntityHandler.java | 82 ++
.../recon/api/handlers/VolumeEntityHandler.java | 160 +++
.../EntityType.java => handlers/package-info.java} | 13 +-
.../ozone/recon/api/types/ContainerKeyPrefix.java | 9 +
.../ozone/recon/api/types/DatanodeMetadata.java | 14 +
.../hadoop/ozone/recon/api/types/EntityType.java | 85 +-
...ainerKeyPrefix.java => KeyPrefixContainer.java} | 50 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 7 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 10 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../ozone/recon/scm/ReconPipelineManager.java | 11 +-
.../recon/scm/ReconPipelineReportHandler.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 14 +-
.../recon/spi/ReconContainerMetadataManager.java | 17 +
.../recon/spi/impl/KeyPrefixContainerCodec.java | 92 ++
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
.../impl/ReconContainerMetadataManagerImpl.java | 304 +++--
.../ozone/recon/spi/impl/ReconDBDefinition.java | 12 +-
.../ozone/recon/spi/impl/ReconDBProvider.java | 11 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 94 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 206 +---
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 217 ++++
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 18 +-
.../src/views/pipelines/pipelines.tsx | 29 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 44 +-
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 72 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 90 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 663 -----------
.../recon/api/TestNSSummaryEndpointWithFSO.java | 1249 ++++++++++++++++++++
.../ozone/recon/api/TestTaskStatusService.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 45 +-
.../recon/fsck/TestContainerHealthStatus.java | 12 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 16 +-
.../TestContainerHealthTaskRecordGenerator.java | 14 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 2 +
.../TestReconInternalSchemaDefinition.java | 21 +-
.../recon/persistence/TestSqlSchemaSetup.java | 6 +-
.../persistence/TestStatsSchemaDefinition.java | 29 +-
.../TestUtilizationSchemaDefinition.java | 40 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 11 +
.../scm/AbstractReconContainerManagerTest.java | 27 +-
.../ozone/recon/scm/TestReconContainerManager.java | 42 +-
...TestReconIncrementalContainerReportHandler.java | 16 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +-
.../recon/scm/TestReconPipelineReportHandler.java | 6 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 21 +-
.../TestReconContainerMetadataManagerImpl.java | 25 +
.../TestStorageContainerServiceProviderImpl.java | 12 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 10 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 541 ---------
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 604 ++++++++++
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 9 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 44 +-
.../ozone/recon/tasks/TestTableCountTask.java | 10 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 24 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 42 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 205 ++--
.../ozone/s3/signature/AWSSignatureProcessor.java | 30 +-
.../hadoop/ozone/s3/signature/AuthOperation.java} | 36 +-
.../s3/signature/AuthorizationV2HeaderParser.java | 12 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 92 +-
.../s3/signature/AuthorizationV4QueryParser.java | 3 +-
.../hadoop/ozone/s3/signature/Credential.java | 14 +-
.../s3/signature/MalformedResourceException.java | 24 +-
.../hadoop/ozone/s3/signature/SignatureParser.java | 4 +-
.../apache/hadoop/ozone/s3/util/AuditUtils.java | 53 +
.../hadoop/ozone/client/ClientProtocolStub.java | 573 +++++++++
.../hadoop/ozone/client/ObjectStoreStub.java | 1 +
.../hadoop/ozone/client/OzoneBucketStub.java | 6 +-
.../hadoop/ozone/client/OzoneClientStub.java | 7 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 28 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 153 ++-
.../ozone/s3/endpoint/TestPermissionCheck.java | 28 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 10 +
.../signature/TestAuthorizationV2HeaderParser.java | 70 +-
.../signature/TestAuthorizationV4HeaderParser.java | 98 +-
.../s3/signature/TestStringToSignProducer.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 5 +-
.../ozone/admin/om/UpdateRangerSubcommand.java | 96 ++
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 73 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 3 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 3 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 103 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 6 +-
.../ozone/debug/container/ContainerCommands.java | 18 +-
.../ozone/debug/container/ExportSubcommand.java | 35 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 10 +-
.../ozone/freon/ClosedContainerReplicator.java | 33 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 5 +-
.../ozone/freon/FreonReplicationOptions.java | 3 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 2 +-
.../ozone/freon/OmBucketReadWriteKeyOps.java | 14 +-
.../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 120 ++
.../ozone/freon/OzoneClientKeyGenerator.java | 2 +-
.../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +-
.../hadoop/ozone/freon/S3BucketGenerator.java | 73 ++
.../hadoop/ozone/freon/S3EntityGenerator.java | 65 +
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 47 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 5 +-
.../freon/containergenerator/GeneratorOm.java | 14 +-
pom.xml | 83 +-
679 files changed, 22045 insertions(+), 11053 deletions(-)
delete mode 100644 dev-support/annotations/pom.xml
delete mode 100644
dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
create mode 100644 hadoop-hdds/annotations/pom.xml
create mode 100644
hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/ReplicateAnnotationProcessor.java
rename {dev-support =>
hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
(100%)
copy {dev-support =>
hadoop-hdds}/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
(100%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestContainerClientMetrics.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeChoosingUtil.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/AbstractContainerScanner.java
rename
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/{ContainerMetadataScrubberMetrics.java
=> AbstractContainerScannerMetrics.java} (68%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScrubberMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScannerMetrics.java
rename
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/{ContainerScrubberConfiguration.java
=> ContainerScannerConfiguration.java} (92%)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
rename
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/{TestContainerScrubberConfiguration.java
=> TestContainerScannerConfiguration.java} (80%)
rename
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/{TestContainerScrubberMetrics.java
=> TestContainerScannerMetrics.java} (89%)
create mode 100644 hadoop-hdds/docs/content/feature/PrefixFSO.zh.md
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneAdmins.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
(62%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
(63%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java
(63%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java
(62%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java
(59%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java
(60%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java
(51%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java
(63%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java
(60%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java
(50%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java
(53%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java
(63%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java
(54%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
(57%)
copy
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java
(60%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AbstractOverReplicationHandler.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ =>
replication}/ContainerReplicaCount.java (92%)
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ =>
replication}/ECContainerReplicaCount.java (93%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECOverReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/OverReplicatedProcessor.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnderReplicatedProcessor.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{UnderReplicationHandler.java
=> UnhealthyReplicationHandler.java} (83%)
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/{states
=> replication}/TestECContainerReplicaCount.java (87%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECOverReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestOverReplicatedProcessor.java
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/{states
=> replication}/TestRatisContainerReplicaCount.java (96%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestUnderReplicatedProcessor.java
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/HadoopRpcOMFailoverProxyProvider.java
rename
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/{OMFailoverProxyProvider.java
=> OMFailoverProxyProviderBase.java} (65%)
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java
delete mode 100644
hadoop-ozone/dist/src/main/compose/ozonesecure/mockserverInitialization.json
create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/echoRPCLoad.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/{scrubber/TestDataScrubber.java
=> scanner/TestDataScanner.java} (95%)
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{TestOmLDBCli.java
=> TestLDBCli.java} (54%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/RangerUserRequest.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantAccessAuthorizerRangerPlugin.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestResetDeletedBlockRetryCountShell.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator/OMHashCodeGenerator.java
(70%)
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator/StringOMHashCodeGeneratorImpl.java
(70%)
copy {dev-support/annotations/src/main/java/org/apache/ozone/annotations =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/hashcodegenerator}/package-info.java
(81%)
rename hadoop-ozone/ozone-manager/src/{test =>
main}/java/org/apache/hadoop/ozone/om/multitenant/InMemoryMultiTenantAccessController.java
(72%)
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizer.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizerDummyPlugin.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/MultiTenantAccessAuthorizerRangerPlugin.java
copy
hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/EntityType.java
=>
ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMEchoRPCRequest.java}
(54%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/{types/EntityType.java
=> handlers/package-info.java} (81%)
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{ContainerKeyPrefix.java
=> KeyPrefixContainer.java} (65%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/KeyPrefixContainerCodec.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
delete mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
delete mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java
copy
hadoop-ozone/{client/src/main/java/org/apache/hadoop/ozone/client/checksum/AbstractBlockChecksumComputer.java
=>
s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthOperation.java}
(53%)
rename
dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
=>
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/MalformedResourceException.java
(61%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/AuditUtils.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/UpdateRangerSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ResetDeletedBlockRetryCountSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmRPCLoadGenerator.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
